Barry Warsaw pushed to branch master at mailman / Mailman
Commits: c9464cb6 by Barry Warsaw at 2015-12-22T14:54:46Z Refactor bump_digest_number_and_volume() and maybe_send_digest_now() into their own module inside the mailman.app package. With the latter, remove the "all lists" functionality and require the mlist argument. We'll handle the "all lists" use case higher up the stack. Also, rename the send-digests handler digests since we'll next add the bump functionality. - - - - - 8d74a4f3 by Barry Warsaw at 2015-12-22T15:38:42Z Port bump-digests to MM3, via the `mailman digests` command. Fix a buglet where if maybe_send_digest_now() was called before a digest has been prepared, a FileNotFoundError would be raised. - - - - - 85521fac by Barry Warsaw at 2015-12-22T17:05:17Z Expose the "bump digest" and "send digest" functionality though the REST API via the ``<api>/lists/<list-id>/digest`` end-point. GETting this resource returns the ``next_digest_number`` and ``volume`` as the same values accessible through the list's configuraiton resource. POSTing to the resource with either ``send=True``, ``bump=True``, or both invokes the given action. - - - - - ea1d7f36 by Barry Warsaw at 2015-12-22T18:26:59Z Bump coverage to 100% - - - - - 10 changed files: - − port_me/bumpdigests.py - + src/mailman/app/digests.py - + src/mailman/app/tests/test_digests.py - src/mailman/commands/cli_send_digests.py → src/mailman/commands/cli_digests.py - src/mailman/commands/tests/test_send_digests.py → src/mailman/commands/tests/test_digests.py - src/mailman/docs/NEWS.rst - src/mailman/handlers/to_digest.py - src/mailman/rest/docs/lists.rst - src/mailman/rest/lists.py - src/mailman/rest/tests/test_lists.py Changes: ===================================== port_me/bumpdigests.py deleted ===================================== --- a/port_me/bumpdigests.py +++ /dev/null @@ -1,74 +0,0 @@ -# Copyright (C) 1998-2015 by the Free Software Foundation, Inc. -# -# This file is part of GNU Mailman. -# -# GNU Mailman is free software: you can redistribute it and/or modify it under -# the terms of the GNU General Public License as published by the Free -# Software Foundation, either version 3 of the License, or (at your option) -# any later version. -# -# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for -# more details. -# -# You should have received a copy of the GNU General Public License along with -# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. - -import sys -import optparse - -from mailman import errors -from mailman import MailList -from mailman.configuration import config -from mailman.core.i18n import _ -from mailman.version import MAILMAN_VERSION - -# Work around known problems with some RedHat cron daemons -import signal -signal.signal(signal.SIGCHLD, signal.SIG_DFL) - - - -def parseargs(): - parser = optparse.OptionParser(version=MAILMAN_VERSION, - usage=_("""\ -%prog [options] [listname ...] - -Increment the digest volume number and reset the digest number to one. All -the lists named on the command line are bumped. If no list names are given, -all lists are bumped.""")) - parser.add_option('-C', '--config', - help=_('Alternative configuration file to use')) - opts, args = parser.parse_args() - return opts, args, parser - - - -def main(): - opts, args, parser = parseargs() - config.load(opts.config) - - listnames = set(args or config.list_manager.names) - if not listnames: - print(_('Nothing to do.')) - sys.exit(0) - - for listname in listnames: - try: - # Be sure the list is locked - mlist = MailList.MailList(listname) - except errors.MMListError: - parser.print_help() - print(_('No such list: $listname'), file=sys.stderr) - sys.exit(1) - try: - mlist.bump_digest_volume() - finally: - mlist.Save() - mlist.Unlock() - - - -if __name__ == '__main__': - main() ===================================== src/mailman/app/digests.py ===================================== --- /dev/null +++ b/src/mailman/app/digests.py @@ -0,0 +1,119 @@ +# Copyright (C) 2015 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. + +"""Digest functions.""" + +__all__ = [ + 'bump_digest_number_and_volume', + 'maybe_send_digest_now', + ] + + +import os + +from mailman.config import config +from mailman.email.message import Message +from mailman.interfaces.digests import DigestFrequency +from mailman.utilities.datetime import now as right_now + + + +def bump_digest_number_and_volume(mlist): + """Bump the digest number and volume.""" + now = right_now() + if mlist.digest_last_sent_at is None: + # There has been no previous digest. + bump = False + elif mlist.digest_volume_frequency == DigestFrequency.yearly: + bump = (now.year > mlist.digest_last_sent_at.year) + elif mlist.digest_volume_frequency == DigestFrequency.monthly: + # Monthly. + this_month = now.year * 100 + now.month + digest_month = (mlist.digest_last_sent_at.year * 100 + + mlist.digest_last_sent_at.month) + bump = (this_month > digest_month) + elif mlist.digest_volume_frequency == DigestFrequency.quarterly: + # Quarterly. + this_quarter = now.year * 100 + (now.month - 1) // 4 + digest_quarter = (mlist.digest_last_sent_at.year * 100 + + (mlist.digest_last_sent_at.month - 1) // 4) + bump = (this_quarter > digest_quarter) + elif mlist.digest_volume_frequency == DigestFrequency.weekly: + this_week = now.year * 100 + now.isocalendar()[1] + digest_week = (mlist.digest_last_sent_at.year * 100 + + mlist.digest_last_sent_at.isocalendar()[1]) + bump = (this_week > digest_week) + elif mlist.digest_volume_frequency == DigestFrequency.daily: + bump = (now.toordinal() > mlist.digest_last_sent_at.toordinal()) + else: + raise AssertionError( + 'Bad DigestFrequency: {0}'.format( + mlist.digest_volume_frequency)) + if bump: + mlist.volume += 1 + mlist.next_digest_number = 1 + else: + # Just bump the digest number. + mlist.next_digest_number += 1 + mlist.digest_last_sent_at = now + + + +def maybe_send_digest_now(mlist, *, force=False): + """Send this mailing list's digest now. + + If there are any messages in this mailing list's digest, the + digest is sent immediately, regardless of whether the size + threshold has been met. When called through the subcommand + `mailman send_digest` the value of .digest_send_periodic is + consulted. + + :param mlist: The mailing list whose digest should be sent. + :type mlist: IMailingList + :param force: Should the digest be sent even if the size threshold hasn't + been met? + :type force: boolean + """ + mailbox_path = os.path.join(mlist.data_path, 'digest.mmdf') + # Calculate the current size of the mailbox file. This will not tell + # us exactly how big the resulting MIME and rfc1153 digest will + # actually be, but it's the most easily available metric to decide + # whether the size threshold has been reached. + try: + size = os.path.getsize(mailbox_path) + except FileNotFoundError: + size = 0 + if (size >= mlist.digest_size_threshold * 1024.0 or + (force and size > 0)): + # Send the digest. Because we don't want to hold up this process + # with crafting the digest, we're going to move the digest file to + # a safe place, then craft a fake message for the DigestRunner as + # a trigger for it to build and send the digest. + mailbox_dest = os.path.join( + mlist.data_path, + 'digest.{0.volume}.{0.next_digest_number}.mmdf'.format( + mlist)) + volume = mlist.volume + digest_number = mlist.next_digest_number + bump_digest_number_and_volume(mlist) + os.rename(mailbox_path, mailbox_dest) + config.switchboards['digest'].enqueue( + Message(), + listid=mlist.list_id, + digest_path=mailbox_dest, + volume=volume, + digest_number=digest_number) ===================================== src/mailman/app/tests/test_digests.py ===================================== --- /dev/null +++ b/src/mailman/app/tests/test_digests.py @@ -0,0 +1,237 @@ +# Copyright (C) 2015 by the Free Software Foundation, Inc. +# +# This file is part of GNU Mailman. +# +# GNU Mailman is free software: you can redistribute it and/or modify it under +# the terms of the GNU General Public License as published by the Free +# Software Foundation, either version 3 of the License, or (at your option) +# any later version. +# +# GNU Mailman is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +# more details. +# +# You should have received a copy of the GNU General Public License along with +# GNU Mailman. If not, see <http://www.gnu.org/licenses/>. + +"""Digest helper tests.""" + +__all__ = [ + 'TestBumpDigest', + 'TestMaybeSendDigest', + ] + + +import os +import unittest + +from datetime import timedelta +from mailman.app.digests import ( + bump_digest_number_and_volume, maybe_send_digest_now) +from mailman.app.lifecycle import create_list +from mailman.config import config +from mailman.interfaces.digests import DigestFrequency +from mailman.interfaces.member import DeliveryMode +from mailman.runners.digest import DigestRunner +from mailman.testing.helpers import ( + get_queue_messages, make_testable_runner, + specialized_message_from_string as mfs, subscribe) +from mailman.testing.layers import ConfigLayer +from mailman.utilities.datetime import factory, now as right_now + + + +class TestBumpDigest(unittest.TestCase): + layer = ConfigLayer + + def setUp(self): + self._mlist = create_list('a...@example.com') + self._mlist.volume = 7 + self._mlist.next_digest_number = 4 + self.right_now = right_now() + + def test_bump_no_previous_digest(self): + self._mlist.digest_last_sent_at = None + bump_digest_number_and_volume(self._mlist) + self.assertEqual(self._mlist.volume, 7) + self.assertEqual(self._mlist.next_digest_number, 5) + self.assertEqual(self._mlist.digest_last_sent_at, self.right_now) + + def test_bump_yearly(self): + self._mlist.digest_last_sent_at = self.right_now + timedelta( + days=-370) + self._mlist.digest_volume_frequency = DigestFrequency.yearly + bump_digest_number_and_volume(self._mlist) + self.assertEqual(self._mlist.volume, 8) + self.assertEqual(self._mlist.next_digest_number, 1) + self.assertEqual(self._mlist.digest_last_sent_at, self.right_now) + + def test_bump_yearly_not_yet(self): + self._mlist.digest_last_sent_at = self.right_now + timedelta( + days=-200) + self._mlist.digest_volume_frequency = DigestFrequency.yearly + bump_digest_number_and_volume(self._mlist) + self.assertEqual(self._mlist.volume, 7) + self.assertEqual(self._mlist.next_digest_number, 5) + self.assertEqual(self._mlist.digest_last_sent_at, self.right_now) + + def test_bump_monthly(self): + self._mlist.digest_last_sent_at = self.right_now + timedelta( + days=-32) + self._mlist.digest_volume_frequency = DigestFrequency.monthly + bump_digest_number_and_volume(self._mlist) + self.assertEqual(self._mlist.volume, 8) + self.assertEqual(self._mlist.next_digest_number, 1) + self.assertEqual(self._mlist.digest_last_sent_at, self.right_now) + + def test_bump_monthly_not_yet(self): + # The normal test date starts on the first day of the month, so let's + # fast forward it a few days so we can set the digest last sent time + # to earlier in the same month. + self._mlist.digest_last_sent_at = self.right_now + factory.fast_forward(days=26) + self._mlist.digest_volume_frequency = DigestFrequency.monthly + bump_digest_number_and_volume(self._mlist) + self.assertEqual(self._mlist.volume, 7) + self.assertEqual(self._mlist.next_digest_number, 5) + self.assertEqual(self._mlist.digest_last_sent_at, right_now()) + + def test_bump_quarterly(self): + self._mlist.digest_last_sent_at = self.right_now + timedelta( + days=-93) + self._mlist.digest_volume_frequency = DigestFrequency.quarterly + bump_digest_number_and_volume(self._mlist) + self.assertEqual(self._mlist.volume, 8) + self.assertEqual(self._mlist.next_digest_number, 1) + self.assertEqual(self._mlist.digest_last_sent_at, self.right_now) + + def test_bump_quarterly_not_yet(self): + self._mlist.digest_last_sent_at = self.right_now + timedelta( + days=-88) + self._mlist.digest_volume_frequency = DigestFrequency.quarterly + bump_digest_number_and_volume(self._mlist) + self.assertEqual(self._mlist.volume, 7) + self.assertEqual(self._mlist.next_digest_number, 5) + self.assertEqual(self._mlist.digest_last_sent_at, self.right_now) + + def test_bump_weekly(self): + self._mlist.digest_last_sent_at = self.right_now + timedelta( + days=-8) + self._mlist.digest_volume_frequency = DigestFrequency.weekly + bump_digest_number_and_volume(self._mlist) + self.assertEqual(self._mlist.volume, 8) + self.assertEqual(self._mlist.next_digest_number, 1) + self.assertEqual(self._mlist.digest_last_sent_at, self.right_now) + + def test_bump_weekly_not_yet(self): + # The normal test date starts on the first day of the week, so let's + # fast forward it a few days so we can set the digest last sent time + # to earlier in the same week. + self._mlist.digest_last_sent_at = self.right_now + factory.fast_forward(days=3) + self._mlist.digest_volume_frequency = DigestFrequency.weekly + bump_digest_number_and_volume(self._mlist) + self.assertEqual(self._mlist.volume, 7) + self.assertEqual(self._mlist.next_digest_number, 5) + self.assertEqual(self._mlist.digest_last_sent_at, right_now()) + + def test_bump_daily(self): + self._mlist.digest_last_sent_at = self.right_now + timedelta( + hours=-27) + self._mlist.digest_volume_frequency = DigestFrequency.daily + bump_digest_number_and_volume(self._mlist) + self.assertEqual(self._mlist.volume, 8) + self.assertEqual(self._mlist.next_digest_number, 1) + self.assertEqual(self._mlist.digest_last_sent_at, self.right_now) + + def test_bump_daily_not_yet(self): + self._mlist.digest_last_sent_at = self.right_now + timedelta( + hours=-5) + self._mlist.digest_volume_frequency = DigestFrequency.daily + bump_digest_number_and_volume(self._mlist) + self.assertEqual(self._mlist.volume, 7) + self.assertEqual(self._mlist.next_digest_number, 5) + self.assertEqual(self._mlist.digest_last_sent_at, self.right_now) + + def test_bump_bad_frequency(self): + self._mlist.digest_last_sent_at = self.right_now + timedelta( + hours=-22) + self._mlist.digest_volume_frequency = -10 + self.assertRaises(AssertionError, + bump_digest_number_and_volume, self._mlist) + + + +class TestMaybeSendDigest(unittest.TestCase): + layer = ConfigLayer + + def setUp(self): + self._mlist = create_list('a...@example.com') + self._mlist.send_welcome_message = False + self._mailbox_path = os.path.join(self._mlist.data_path, 'digest.mmdf') + # The mailing list needs at least one digest recipient. + member = subscribe(self._mlist, 'Anne') + member.preferences.delivery_mode = DeliveryMode.plaintext_digests + self._subject_number = 1 + self._runner = make_testable_runner(DigestRunner, 'digest') + + def _to_digest(self, count=1): + for i in range(count): + msg = mfs("""\ +To: a...@example.com +From: a...@example.com +Subject: message {} + +""".format(self._subject_number)) + self._subject_number += 1 + config.handlers['to-digest'].process(self._mlist, msg, {}) + + def test_send_digest_over_threshold(self): + # Put a few messages in the digest. + self._to_digest(3) + # Set the size threshold low enough to trigger a send. + self._mlist.digest_size_threshold = 0.1 + maybe_send_digest_now(self._mlist) + self._runner.run() + # There are no digests in flight now, and a single digest message has + # been sent. + self.assertEqual(len(get_queue_messages('digest')), 0) + self.assertFalse(os.path.exists(self._mailbox_path)) + items = get_queue_messages('virgin') + self.assertEqual(len(items), 1) + digest_contents = str(items[0].msg) + self.assertIn('Subject: message 1', digest_contents) + self.assertIn('Subject: message 2', digest_contents) + + def test_dont_send_digest_under_threshold(self): + # Put a few messages in the digest. + self._to_digest(3) + # Set the size threshold high enough to not trigger a send. + self._mlist.digest_size_threshold = 100 + maybe_send_digest_now(self._mlist) + self._runner.run() + # A digest is still being collected, but none have been sent. + self.assertEqual(len(get_queue_messages('digest')), 0) + self.assertGreater(os.path.getsize(self._mailbox_path), 0) + self.assertLess(os.path.getsize(self._mailbox_path), 100 * 1024.0) + items = get_queue_messages('virgin') + self.assertEqual(len(items), 0) + + def test_force_send_digest_under_threshold(self): + # Put a few messages in the digest. + self._to_digest(3) + # Set the size threshold high enough to not trigger a send. + self._mlist.digest_size_threshold = 100 + # Force sending a digest anyway. + maybe_send_digest_now(self._mlist, force=True) + self._runner.run() + # There are no digests in flight now, and a single digest message has + # been sent. + self.assertEqual(len(get_queue_messages('digest')), 0) + self.assertFalse(os.path.exists(self._mailbox_path)) + items = get_queue_messages('virgin') + self.assertEqual(len(items), 1) + digest_contents = str(items[0].msg) + self.assertIn('Subject: message 1', digest_contents) + self.assertIn('Subject: message 2', digest_contents) ===================================== src/mailman/commands/cli_send_digests.py → src/mailman/commands/cli_digests.py ===================================== --- a/src/mailman/commands/cli_send_digests.py +++ b/src/mailman/commands/cli_digests.py @@ -18,14 +18,15 @@ """The `send_digests` subcommand.""" __all__ = [ - 'Send', + 'Digests', ] import sys +from mailman.app.digests import ( + bump_digest_number_and_volume, maybe_send_digest_now) from mailman.core.i18n import _ -from mailman.handlers.to_digest import maybe_send_digest_now from mailman.interfaces.command import ICLISubCommand from mailman.interfaces.listmanager import IListManager from zope.component import getUtility @@ -34,10 +35,10 @@ from zope.interface import implementer @implementer(ICLISubCommand) -class Send: - """Send some mailing list digests right now.""" +class Digests: + """Operate on digests.""" - name = 'send-digests' + name = 'digests' def add(self, parser, command_parser): """See `ICLISubCommand`.""" @@ -45,25 +46,42 @@ class Send: command_parser.add_argument( '-l', '--list', default=[], dest='lists', metavar='list', action='append', - help=_("""Send the digests for this mailing list. Multiple --list + help=_("""Operate on this mailing list. Multiple --list options can be given. The argument can either be a List-ID - or a fully qualified list name. Without this option, the - digests for all mailing lists will be sent if possible.""")) + or a fully qualified list name. Without this option, + operate on the digests for all mailing lists.""")) + command_parser.add_argument( + '-s', '--send', + default=False, action='store_true', + help=_("""Send any collected digests right now, even if the size + threshold has not yet been met.""")) + command_parser.add_argument( + '-b', '--bump', + default=False, action='store_true', + help=_("""Increment the digest volume number and reset the digest + number to one. If given with --send, the volume number is + incremented before any current digests are sent.""")) def process(self, args): """See `ICLISubCommand`.""" - if not args.lists: - # Send the digests for every list. - maybe_send_digest_now(force=True) - return list_manager = getUtility(IListManager) - for list_spec in args.lists: - # We'll accept list-ids or fqdn list names. - if '@' in list_spec: - mlist = list_manager.get(list_spec) - else: - mlist = list_manager.get_by_list_id(list_spec) - if mlist is None: - print(_('No such list found: $list_spec'), file=sys.stderr) - continue - maybe_send_digest_now(mlist, force=True) + if args.lists: + lists = [] + for spec in args.lists: + # We'll accept list-ids or fqdn list names. + if '@' in spec: + mlist = list_manager.get(spec) + else: + mlist = list_manager.get_by_list_id(spec) + if mlist is None: + print(_('No such list found: $spec'), file=sys.stderr) + else: + lists.append(mlist) + else: + lists = list(list_manager.mailing_lists) + if args.bump: + for mlist in lists: + bump_digest_number_and_volume(mlist) + if args.send: + for mlist in lists: + maybe_send_digest_now(mlist, force=True) ===================================== src/mailman/commands/tests/test_send_digests.py → src/mailman/commands/tests/test_digests.py ===================================== --- a/src/mailman/commands/tests/test_send_digests.py +++ b/src/mailman/commands/tests/test_digests.py @@ -18,6 +18,7 @@ """Test the send-digests subcommand.""" __all__ = [ + 'TestBumpVolume', 'TestSendDigests', ] @@ -25,16 +26,19 @@ __all__ = [ import os import unittest +from datetime import timedelta from io import StringIO from mailman.app.lifecycle import create_list -from mailman.commands.cli_send_digests import Send +from mailman.commands.cli_digests import Digests from mailman.config import config +from mailman.interfaces.digests import DigestFrequency from mailman.interfaces.member import DeliveryMode from mailman.runners.digest import DigestRunner from mailman.testing.helpers import ( get_queue_messages, make_testable_runner, specialized_message_from_string as mfs, subscribe) from mailman.testing.layers import ConfigLayer +from mailman.utilities.datetime import now as right_now from unittest.mock import patch @@ -42,12 +46,12 @@ from unittest.mock import patch class FakeArgs: def __init__(self): self.lists = [] + self.send = False + self.bump = False class TestSendDigests(unittest.TestCase): - """Test the send-digests subcommand.""" - layer = ConfigLayer def setUp(self): @@ -55,7 +59,7 @@ class TestSendDigests(unittest.TestCase): self._mlist.digests_enabled = True self._mlist.digest_size_threshold = 100000 self._mlist.send_welcome_message = False - self._command = Send() + self._command = Digests() self._handler = config.handlers['to-digest'] self._runner = make_testable_runner(DigestRunner, 'digest') # The mailing list needs at least one digest recipient. @@ -80,6 +84,7 @@ Subject: message 1 mailbox_path = os.path.join(self._mlist.data_path, 'digest.mmdf') self.assertGreater(os.path.getsize(mailbox_path), 0) args = FakeArgs() + args.send = True args.lists.append('ant.example.com') self._command.process(args) self._runner.run() @@ -110,6 +115,7 @@ Subject: message 1 mailbox_path = os.path.join(self._mlist.data_path, 'digest.mmdf') self.assertGreater(os.path.getsize(mailbox_path), 0) args = FakeArgs() + args.send = True args.lists.append('a...@example.com') self._command.process(args) self._runner.run() @@ -140,9 +146,10 @@ Subject: message 1 mailbox_path = os.path.join(self._mlist.data_path, 'digest.mmdf') self.assertGreater(os.path.getsize(mailbox_path), 0) args = FakeArgs() + args.send = True args.lists.append('bee.example.com') stderr = StringIO() - with patch('mailman.commands.cli_send_digests.sys.stderr', stderr): + with patch('mailman.commands.cli_digests.sys.stderr', stderr): self._command.process(args) self._runner.run() # The warning was printed to stderr. @@ -171,9 +178,10 @@ Subject: message 1 mailbox_path = os.path.join(self._mlist.data_path, 'digest.mmdf') self.assertGreater(os.path.getsize(mailbox_path), 0) args = FakeArgs() + args.send = True args.lists.append('b...@example.com') stderr = StringIO() - with patch('mailman.commands.cli_send_digests.sys.stderr', stderr): + with patch('mailman.commands.cli_digests.sys.stderr', stderr): self._command.process(args) self._runner.run() # The warning was printed to stderr. @@ -202,9 +210,10 @@ Subject: message 1 mailbox_path = os.path.join(self._mlist.data_path, 'digest.mmdf') self.assertGreater(os.path.getsize(mailbox_path), 0) args = FakeArgs() + args.send = True args.lists.extend(('ant.example.com', 'bee.example.com')) stderr = StringIO() - with patch('mailman.commands.cli_send_digests.sys.stderr', stderr): + with patch('mailman.commands.cli_digests.sys.stderr', stderr): self._command.process(args) self._runner.run() # The warning was printed to stderr. @@ -260,6 +269,7 @@ Subject: message 3 self.assertEqual(len(items), 0) # Process both list's digests. args = FakeArgs() + args.send = True args.lists.extend(('ant.example.com', 'b...@example.com')) self._command.process(args) self._runner.run() @@ -327,7 +337,9 @@ Subject: message 3 items = get_queue_messages('digest') self.assertEqual(len(items), 0) # Process all mailing list digests by not setting any arguments. - self._command.process(FakeArgs()) + args = FakeArgs() + args.send = True + self._command.process(args) self._runner.run() # Now, neither list has a digest mbox and but there are plaintext # digest in the outgoing queue for both. @@ -351,3 +363,86 @@ Subject: message 3 digest_contents = str(bee) self.assertIn('Subject: message 3', digest_contents) self.assertIn('Subject: message 4', digest_contents) + + def test_send_no_digest_ready(self): + # If no messages have been sent through the mailing list, no digest + # can be sent. + mailbox_path = os.path.join(self._mlist.data_path, 'digest.mmdf') + self.assertFalse(os.path.exists(mailbox_path)) + args = FakeArgs() + args.send = True + args.lists.append('ant.example.com') + self._command.process(args) + self._runner.run() + items = get_queue_messages('virgin') + self.assertEqual(len(items), 0) + + def test_bump_before_send(self): + self._mlist.digest_volume_frequency = DigestFrequency.monthly + self._mlist.volume = 7 + self._mlist.next_digest_number = 4 + self._mlist.digest_last_sent_at = right_now() + timedelta( + days=-32) + msg = mfs("""\ +To: a...@example.com +From: a...@example.com +Subject: message 1 + +""") + self._handler.process(self._mlist, msg, {}) + args = FakeArgs() + args.bump = True + args.send = True + args.lists.append('ant.example.com') + self._command.process(args) + self._runner.run() + # The volume is 8 and the digest number is 2 because a digest was sent + # after the volume/number was bumped. + self.assertEqual(self._mlist.volume, 8) + self.assertEqual(self._mlist.next_digest_number, 2) + self.assertEqual(self._mlist.digest_last_sent_at, right_now()) + items = get_queue_messages('virgin') + self.assertEqual(len(items), 1) + self.assertEqual(items[0].msg['subject'], 'Ant Digest, Vol 8, Issue 1') + + + +class TestBumpVolume(unittest.TestCase): + layer = ConfigLayer + + def setUp(self): + self._mlist = create_list('a...@example.com') + self._mlist.digest_volume_frequency = DigestFrequency.monthly + self._mlist.volume = 7 + self._mlist.next_digest_number = 4 + self.right_now = right_now() + self._command = Digests() + + def test_bump_one_list(self): + self._mlist.digest_last_sent_at = self.right_now + timedelta( + days=-32) + args = FakeArgs() + args.bump = True + args.lists.append('ant.example.com') + self._command.process(args) + self.assertEqual(self._mlist.volume, 8) + self.assertEqual(self._mlist.next_digest_number, 1) + self.assertEqual(self._mlist.digest_last_sent_at, self.right_now) + + def test_bump_two_lists(self): + self._mlist.digest_last_sent_at = self.right_now + timedelta( + days=-32) + # Create the second list. + bee = create_list('b...@example.com') + bee.digest_volume_frequency = DigestFrequency.monthly + bee.volume = 7 + bee.next_digest_number = 4 + bee.digest_last_sent_at = self.right_now + timedelta( + days=-32) + args = FakeArgs() + args.bump = True + args.lists.extend(('ant.example.com', 'bee.example.com')) + self._command.process(args) + self.assertEqual(self._mlist.volume, 8) + self.assertEqual(self._mlist.next_digest_number, 1) + self.assertEqual(self._mlist.digest_last_sent_at, self.right_now) ===================================== src/mailman/docs/NEWS.rst ===================================== --- a/src/mailman/docs/NEWS.rst +++ b/src/mailman/docs/NEWS.rst @@ -123,6 +123,12 @@ REST * Expose ``digest_send_periodic``, ``digest_volume_frequency``, and ``digests_enabled`` (renamed from ``digestable``) to the REST API. (Closes: #159) + * Expose the "bump digest" and "send digest" functionality though the REST + API via the ``<api>/lists/<list-id>/digest`` end-point. GETting this + resource returns the ``next_digest_number`` and ``volume`` as the same + values accessible through the list's configuraiton resource. POSTing to + the resource with either ``send=True``, ``bump=True``, or both invokes the + given action. Other ----- @@ -132,9 +138,9 @@ Other ``list_url`` or permalink. Given by Aurélien Bompard. * Large performance improvement in ``SubscriptionService.find_members()``. Given by Aurélien Bompard. - * Rework the digest machinery, and add a new `send-digests` subcommand, which + * Rework the digest machinery, and add a new ``digests`` subcommand, which can be used from the command line or cron to immediately send out any - partially collected digests. + partially collected digests, or bump the digest and volume numbers. * The mailing list "data directory" has been renamed. Instead of using the fqdn listname, the subdirectory inside ``[paths]list_data_dir`` now uses the List-ID. ===================================== src/mailman/handlers/to_digest.py ===================================== --- a/src/mailman/handlers/to_digest.py +++ b/src/mailman/handlers/to_digest.py @@ -19,22 +19,15 @@ __all__ = [ 'ToDigest', - 'bump_digest_number_and_volume', - 'maybe_send_digest_now', ] import os -from mailman.config import config +from mailman.app.digests import maybe_send_digest_now from mailman.core.i18n import _ -from mailman.email.message import Message -from mailman.interfaces.digests import DigestFrequency from mailman.interfaces.handler import IHandler -from mailman.interfaces.listmanager import IListManager -from mailman.utilities.datetime import now as right_now from mailman.utilities.mailbox import Mailbox -from zope.component import getUtility from zope.interface import implementer @@ -58,95 +51,3 @@ class ToDigest: with Mailbox(mailbox_path, create=True) as mbox: mbox.add(msg) maybe_send_digest_now(mlist) - - - -def bump_digest_number_and_volume(mlist): - """Bump the digest number and volume.""" - now = right_now() - if mlist.digest_last_sent_at is None: - # There has been no previous digest. - bump = False - elif mlist.digest_volume_frequency == DigestFrequency.yearly: - bump = (now.year > mlist.digest_last_sent_at.year) - elif mlist.digest_volume_frequency == DigestFrequency.monthly: - # Monthly. - this_month = now.year * 100 + now.month - digest_month = (mlist.digest_last_sent_at.year * 100 + - mlist.digest_last_sent_at.month) - bump = (this_month > digest_month) - elif mlist.digest_volume_frequency == DigestFrequency.quarterly: - # Quarterly. - this_quarter = now.year * 100 + (now.month - 1) // 4 - digest_quarter = (mlist.digest_last_sent_at.year * 100 + - (mlist.digest_last_sent_at.month - 1) // 4) - bump = (this_quarter > digest_quarter) - elif mlist.digest_volume_frequency == DigestFrequency.weekly: - this_week = now.year * 100 + now.isocalendar()[1] - digest_week = (mlist.digest_last_sent_at.year * 100 + - mlist.digest_last_sent_at.isocalendar()[1]) - bump = (this_week > digest_week) - elif mlist.digest_volume_frequency == DigestFrequency.daily: - bump = (now.toordinal() > mlist.digest_last_sent_at.toordinal()) - else: - raise AssertionError( - 'Bad DigestFrequency: {0}'.format( - mlist.digest_volume_frequency)) - if bump: - mlist.volume += 1 - mlist.next_digest_number = 1 - else: - # Just bump the digest number. - mlist.next_digest_number += 1 - mlist.digest_last_sent_at = now - - - -def maybe_send_digest_now(mlist=None, force=False): - """Send this mailing list's digest now. - - If there are any messages in this mailing list's digest, the - digest is sent immediately, regardless of whether the size - threshold has been met. When called through the subcommand - `mailman send_digest` the value of .digest_send_periodic is - consulted. - - :param mlist: The mailing list whose digest should be sent. If this is - None, all mailing lists with non-zero sized digests will have theirs - sent immediately. - :type mlist: IMailingList or None - :param force: Should the digest be sent even if the size threshold hasn't - been met? - :type force: boolean - """ - if mlist is None: - digestable_lists = getUtility(IListManager).mailing_lists - else: - digestable_lists = [mlist] - for mailing_list in digestable_lists: - mailbox_path = os.path.join(mailing_list.data_path, 'digest.mmdf') - # Calculate the current size of the mailbox file. This will not tell - # us exactly how big the resulting MIME and rfc1153 digest will - # actually be, but it's the most easily available metric to decide - # whether the size threshold has been reached. - size = os.path.getsize(mailbox_path) - if (size >= mailing_list.digest_size_threshold * 1024.0 or - (force and size > 0)): - # Send the digest. Because we don't want to hold up this process - # with crafting the digest, we're going to move the digest file to - # a safe place, then craft a fake message for the DigestRunner as - # a trigger for it to build and send the digest. - mailbox_dest = os.path.join( - mailing_list.data_path, - 'digest.{0.volume}.{0.next_digest_number}.mmdf'.format( - mailing_list)) - volume = mailing_list.volume - digest_number = mailing_list.next_digest_number - bump_digest_number_and_volume(mailing_list) - os.rename(mailbox_path, mailbox_dest) - config.switchboards['digest'].enqueue( - Message(), - listid=mailing_list.list_id, - digest_path=mailbox_dest, - volume=volume, - digest_number=digest_number) ===================================== src/mailman/rest/docs/lists.rst ===================================== --- a/src/mailman/rest/docs/lists.rst +++ b/src/mailman/rest/docs/lists.rst @@ -288,3 +288,112 @@ You can change the state of a subset of the list archivers. mail-archive: False mhonarc: False prototype: False + + +List digests +============ + +A list collects messages and prepares a digest which can be periodically sent +to all members who elect to receive digests. Digests are usually sent +whenever their size has reached a threshold, but you can force a digest to be +sent immediately via the REST API. + +Let's create a mailing list that has a digest recipient. + + >>> from mailman.interfaces.member import DeliveryMode + >>> from mailman.testing.helpers import subscribe + >>> emu = create_list('e...@example.com') + >>> emu.send_welcome_message = False + >>> anne = subscribe(emu, 'Anne') + >>> anne.preferences.delivery_mode = DeliveryMode.plaintext_digests + +The mailing list has a fairly high size threshold so that sending a single +message through the list won't trigger an automatic digest. The threshold is +the maximum digest size in kibibytes (1024 bytes). + + >>> emu.digest_size_threshold = 100 + >>> transaction.commit() + +We send a message through the mailing list to start collecting for a digest. + + >>> from mailman.runners.digest import DigestRunner + >>> from mailman.testing.helpers import make_testable_runner + >>> msg = message_from_string("""\ + ... From: a...@example.com + ... To: e...@example.com + ... Subject: Message #1 + ... + ... """) + >>> config.handlers['to-digest'].process(emu, msg, {}) + >>> runner = make_testable_runner(DigestRunner, 'digest') + >>> runner.run() + +No digest was sent because it didn't reach the size threshold. + + >>> from mailman.testing.helpers import get_queue_messages + >>> len(get_queue_messages('virgin')) + 0 + +By POSTing to the list's digest end-point with the ``send`` parameter set, we +can force the digest to be sent. + + >>> dump_json('http://localhost:9001/3.0/lists/emu.example.com/digest', { + ... 'send': True, + ... }) + content-length: 0 + date: ... + +Once the runner does its thing, the digest message will be sent. + + >>> runner.run() + >>> items = get_queue_messages('virgin') + >>> len(items) + 1 + >>> print(items[0].msg) + From: emu-requ...@example.com + Subject: Emu Digest, Vol 1, Issue 1 + To: e...@example.com + ... + From: a...@example.com + Subject: Message #1 + To: e...@example.com + ... + End of Emu Digest, Vol 1, Issue 1 + ********************************* + <BLANKLINE> + +Digests also have a volume number and digest number which can be bumped, also +by POSTing to the REST API. Bumping the digest for this list will increment +the digest volume and reset the digest number to 1. We have to fake that the +last digest was sent a couple of days ago. + + >>> from datetime import timedelta + >>> from mailman.interfaces.digests import DigestFrequency + >>> emu.digest_volume_frequency = DigestFrequency.daily + >>> emu.digest_last_sent_at -= timedelta(days=2) + >>> transaction.commit() + +Before bumping, we can get the next digest volume and number. Doing a GET on +the digest resource is just a shorthand for getting some interesting +information about the digest. Note that ``volume`` and ``next_digest_number`` +can also be retrieved from the list's configuration resource. + + >>> dump_json('http://localhost:9001/3.0/lists/emu.example.com/digest') + http_etag: ... + next_digest_number: 2 + volume: 1 + +Let's bump the digest. + + >>> dump_json('http://localhost:9001/3.0/lists/emu.example.com/digest', { + ... 'bump': True, + ... }) + content-length: 0 + date: ... + +And now the next digest to be sent will have a new volume number. + + >>> dump_json('http://localhost:9001/3.0/lists/emu.example.com/digest') + http_etag: ... + next_digest_number: 1 + volume: 2 ===================================== src/mailman/rest/lists.py ===================================== --- a/src/mailman/rest/lists.py +++ b/src/mailman/rest/lists.py @@ -28,6 +28,8 @@ __all__ = [ from lazr.config import as_boolean +from mailman.app.digests import ( + bump_digest_number_and_volume, maybe_send_digest_now) from mailman.app.lifecycle import create_list, remove_list from mailman.config import config from mailman.interfaces.domain import BadDomainSpecificationError @@ -39,8 +41,8 @@ from mailman.interfaces.styles import IStyleManager from mailman.interfaces.subscriptions import ISubscriptionService from mailman.rest.listconf import ListConfiguration from mailman.rest.helpers import ( - CollectionMixin, GetterSetter, NotFound, bad_request, child, created, - etag, no_content, not_found, okay) + CollectionMixin, GetterSetter, NotFound, accepted, bad_request, child, + created, etag, no_content, not_found, okay) from mailman.rest.members import AMember, MemberCollection from mailman.rest.post_moderation import HeldMessages from mailman.rest.sub_moderation import SubscriptionRequests @@ -192,6 +194,12 @@ class AList(_ListBase): return NotFound(), [] return ListArchivers(self._mlist) + @child() + def digest(self, request, segments): + if self._mlist is None: + return NotFound(), [] + return ListDigest(self._mlist) + class AllLists(_ListBase): @@ -268,8 +276,7 @@ class ArchiverGetterSetter(GetterSetter): # getting a new status. value will be the representation of the new # boolean status. archiver = self._archiver_set.get(attribute) - if archiver is None: - raise ValueError('No such archiver: {}'.format(attribute)) + assert archiver is not None, attribute archiver.is_enabled = as_boolean(value) @@ -310,6 +317,41 @@ class ListArchivers: +class ListDigest: + """Simple resource representing actions on a list's digest.""" + + def __init__(self, mlist): + self._mlist = mlist + + def on_get(self, request, response): + resource = dict( + next_digest_number=self._mlist.next_digest_number, + volume=self._mlist.volume, + ) + okay(response, etag(resource)) + + def on_post(self, request, response): + try: + validator = Validator( + send=as_boolean, + bump=as_boolean, + _optional=('send', 'bump')) + values = validator(request) + except ValueError as error: + bad_request(response, str(error)) + return + if len(values) == 0: + # There's nothing to do, but that's okay. + okay(response) + return + if values.get('bump', False): + bump_digest_number_and_volume(self._mlist) + if values.get('send', False): + maybe_send_digest_now(self._mlist, force=True) + accepted(response) + + + class Styles: """Simple resource representing all list styles.""" ===================================== src/mailman/rest/tests/test_lists.py ===================================== --- a/src/mailman/rest/tests/test_lists.py +++ b/src/mailman/rest/tests/test_lists.py @@ -19,6 +19,7 @@ __all__ = [ 'TestListArchivers', + 'TestListDigests', 'TestListPagination', 'TestLists', 'TestListsMissing', @@ -27,15 +28,22 @@ __all__ = [ import unittest +from datetime import timedelta from mailman.app.lifecycle import create_list from mailman.config import config from mailman.database.transaction import transaction +from mailman.interfaces.digests import DigestFrequency from mailman.interfaces.listmanager import IListManager from mailman.interfaces.mailinglist import IAcceptableAliasSet +from mailman.interfaces.member import DeliveryMode from mailman.interfaces.usermanager import IUserManager from mailman.model.mailinglist import AcceptableAlias -from mailman.testing.helpers import call_api +from mailman.runners.digest import DigestRunner +from mailman.testing.helpers import ( + call_api, get_queue_messages, make_testable_runner, + specialized_message_from_string as mfs) from mailman.testing.layers import RESTLayer +from mailman.utilities.datetime import now as right_now from urllib.error import HTTPError from zope.component import getUtility @@ -192,6 +200,24 @@ class TestLists(unittest.TestCase): self.assertIsNone(getUtility(IListManager).get('t...@example.com')) self.assertEqual(config.db.store.query(AcceptableAlias).count(), 0) + def test_bad_roster_matcher(self): + # Try to get a list's roster, but the roster name is bogus. + with self.assertRaises(HTTPError) as cm: + call_api('http://localhost:9001/3.0/lists/ant.example.com' + '/roster/bogus') + self.assertEqual(cm.exception.code, 404) + + def test_bad_config_matcher(self): + with self.assertRaises(HTTPError) as cm: + call_api('http://localhost:9001/3.0/lists/ant.example.com' + '/config/volume/bogus') + self.assertEqual(cm.exception.code, 404) + + def test_bad_list_get(self): + with self.assertRaises(HTTPError) as cm: + call_api('http://localhost:9001/3.0/lists/bogus.example.com') + self.assertEqual(cm.exception.code, 404) + class TestListArchivers(unittest.TestCase): @@ -222,8 +248,20 @@ class TestListArchivers(unittest.TestCase): 'http://localhost:9001/3.0/lists/bee.example.com/archivers') self.assertEqual(cm.exception.code, 404) - def test_patch_status_on_bogus_archiver(self): - # You cannot set the status on an archiver the list doesn't know about. + def test_put_bogus_archiver(self): + # You cannot PUT on an archiver the list doesn't know about. + with self.assertRaises(HTTPError) as cm: + call_api( + 'http://localhost:9001/3.0/lists/ant.example.com/archivers', { + 'bogus-archiver': True, + }, + method='PUT') + self.assertEqual(cm.exception.code, 400) + self.assertEqual(cm.exception.reason, + b'Unexpected parameters: bogus-archiver') + + def test_patch_bogus_archiver(self): + # You cannot PATCH on an archiver the list doesn't know about. with self.assertRaises(HTTPError) as cm: call_api( 'http://localhost:9001/3.0/lists/ant.example.com/archivers', { @@ -318,7 +356,7 @@ class TestListPagination(unittest.TestCase): def test_zeroth_page(self): # Page numbers start at one. with self.assertRaises(HTTPError) as cm: - resource, response = call_api( + call_api( 'http://localhost:9001/3.0/domains/example.com/lists' '?count=1&page=0') self.assertEqual(cm.exception.code, 400) @@ -326,7 +364,7 @@ class TestListPagination(unittest.TestCase): def test_negative_page(self): # Negative pages are not allowed. with self.assertRaises(HTTPError) as cm: - resource, response = call_api( + call_api( 'http://localhost:9001/3.0/domains/example.com/lists' '?count=1&page=-1') self.assertEqual(cm.exception.code, 400) @@ -340,3 +378,72 @@ class TestListPagination(unittest.TestCase): self.assertEqual(resource['total_size'], 6) self.assertEqual(resource['start'], 6) self.assertNotIn('entries', resource) + + + +class TestListDigests(unittest.TestCase): + """Test /lists/<list-id>/digest""" + + layer = RESTLayer + + def setUp(self): + with transaction(): + self._mlist = create_list('a...@example.com') + self._mlist.send_welcome_message = False + anne = getUtility(IUserManager).create_address('a...@example.com') + self._mlist.subscribe(anne) + anne.preferences.delivery_mode = DeliveryMode.plaintext_digests + + def test_bad_digest_url(self): + with self.assertRaises(HTTPError) as cm: + call_api( + 'http://localhost:9001/3.0/lists/bogus.example.com/digest') + self.assertEqual(cm.exception.code, 404) + + def test_post_nothing_to_do(self): + resource, response = call_api( + 'http://localhost:9001/3.0/lists/ant.example.com/digest', {}) + self.assertEqual(response.status, 200) + + def test_post_something_to_do(self): + resource, response = call_api( + 'http://localhost:9001/3.0/lists/ant.example.com/digest', dict( + bump=True)) + self.assertEqual(response.status, 202) + + def test_post_bad_request(self): + with self.assertRaises(HTTPError) as cm: + call_api( + 'http://localhost:9001/3.0/lists/ant.example.com/digest', dict( + bogus=True)) + self.assertEqual(cm.exception.code, 400) + self.assertEqual(cm.exception.reason, b'Unexpected parameters: bogus') + + def test_bump_before_send(self): + with transaction(): + self._mlist.digest_volume_frequency = DigestFrequency.monthly + self._mlist.volume = 7 + self._mlist.next_digest_number = 4 + self._mlist.digest_last_sent_at = right_now() + timedelta( + days=-32) + msg = mfs("""\ +To: a...@example.com +From: a...@example.com +Subject: message 1 + +""") + config.handlers['to-digest'].process(self._mlist, msg, {}) + resource, response = call_api( + 'http://localhost:9001/3.0/lists/ant.example.com/digest', dict( + send=True, + bump=True)) + self.assertEqual(response.status, 202) + make_testable_runner(DigestRunner, 'digest').run() + # The volume is 8 and the digest number is 2 because a digest was sent + # after the volume/number was bumped. + self.assertEqual(self._mlist.volume, 8) + self.assertEqual(self._mlist.next_digest_number, 2) + self.assertEqual(self._mlist.digest_last_sent_at, right_now()) + items = get_queue_messages('virgin') + self.assertEqual(len(items), 1) + self.assertEqual(items[0].msg['subject'], 'Ant Digest, Vol 8, Issue 1') View it on GitLab: https://gitlab.com/mailman/mailman/compare/8e24476848de89302d9b0a8ea91116288201a95d...ea1d7f360edd8f9ac70be5ed55caeaec27feb128
_______________________________________________ Mailman-checkins mailing list Mailman-checkins@python.org Unsubscribe: https://mail.python.org/mailman/options/mailman-checkins/archive%40jab.org
