Update of /cvsroot/mailman/mailman/Mailman/Queue
In directory sc8-pr-cvs1:/tmp/cvs-serv2259
Modified Files:
BounceRunner.py
Log Message:
Implement a simple queuing bounce processor, which should allow more
efficient -- and more controllable -- bounce processing without
killing lock contention.
__init__(): Two new variables to collect bounce event information.
_bouncecnt keeps a count of all events since the last processing pass
(mostly for logging). _bounces is a dictionary mapping listnames to
sequence of bounce events. A bounce event is a tuple of
(address, day, msg)
where day is (YYYY, MM, DD).
_dispose(): Move all the actual registration of bounces out of this
method. Instead, we add bounce events to the queue, which gets
processed by a _doperiodic() pass. This allows us to never lock any
lists in _dispose().
_doperiodic(): Check periodically for queued bounces that need to be
processed, by default once every 15 minutes.
_cleanup(): Needs to call _register_bounces() to take care of any
queued bounces before the runner is shut down.
_register_bounces(): This does the actual bounce registration,
hopefully in a more efficient manner, by collating all queued bounce
events and locking each list only once. Also handles registering
bounces to the site list.
This stuff still needs a bit more testing.
Index: BounceRunner.py
===================================================================
RCS file: /cvsroot/mailman/mailman/Mailman/Queue/BounceRunner.py,v
retrieving revision 2.27
retrieving revision 2.28
diff -u -d -r2.27 -r2.28
--- BounceRunner.py 2 Dec 2002 14:34:18 -0000 2.27
+++ BounceRunner.py 17 Aug 2003 17:48:41 -0000 2.28
@@ -17,6 +17,8 @@
"""Bounce queue runner."""
import re
+import time
+
from email.MIMEText import MIMEText
from email.MIMEMessage import MIMEMessage
from email.Utils import parseaddr
@@ -33,12 +35,22 @@
COMMASPACE = ', '
+REGISTER_BOUNCES_EVERY = mm_cfg.minutes(15)
+
class BounceRunner(Runner):
QDIR = mm_cfg.BOUNCEQUEUE_DIR
- # We only do bounce processing once per minute.
- SLEEPTIME = mm_cfg.minutes(1)
+
+ def __init__(self, slice=None, numslices=1):
+ Runner.__init__(self, slice, numslices)
+ # This is a simple sequence of bounce score events. Each entry in the
+ # list is a tuple of (address, day, msg) where day is a tuple of
+ # (YYYY, MM, DD). We'll sort and collate all this information in
+ # _register_bounces() below.
+ self._bounces = {}
+ self._bouncecnt = 0
+ self._next_registration = time.time() + REGISTER_BOUNCES_EVERY
def _dispose(self, mlist, msg, msgdata):
# Make sure we have the most up-to-date state
@@ -84,57 +96,47 @@
# although I'm unsure how that could happen. Possibly ScanMessages()
# can let None's sneak through. In any event, this will kill them.
addrs = filter(None, addrs)
- # Okay, we have some recognized addresses. We now need to register
- # the bounces for each of these. If the bounce came to the site list,
- # then we'll register the address on every list in the system, but
- # note: this could be VERY resource intensive!
- foundp = 0
- listname = mlist.internal_name()
- if listname == mm_cfg.MAILMAN_SITE_LIST:
- foundp = 1
- for listname in Utils.list_names():
- xlist = self._open_list(listname)
- xlist.Load()
- for addr in addrs:
- if xlist.isMember(addr):
- unlockp = 0
- if not xlist.Locked():
- try:
- xlist.Lock(timeout=mm_cfg.LIST_LOCK_TIMEOUT)
- except LockFile.TimeOutError:
- # Oh well, forget aboutf this list
- continue
- unlockp = 1
- try:
- xlist.registerBounce(addr, msg)
- foundp = 1
- xlist.Save()
- finally:
- if unlockp:
- xlist.Unlock()
+ # Store the bounce score events so we can register them periodically
+ today = time.localtime()[:3]
+ events = [(addr, today, msg) for addr in addrs]
+ self._bounces.setdefault(listname, []).extend(events)
+ self._bouncecnt += len(addrs)
+
+ def _doperiodic(self):
+ now = time.time()
+ if self._next_registration > now or not self._bounces:
+ return
+ # Let's go ahead and register the bounces we've got stored up
+ self._next_registration = now + REGISTER_BOUNCES_EVERY
+ self._register_bounces()
+
+ def _register_bounces(self):
+ syslog('bounce', 'Processing %s queued bounces', self._bouncecnt)
+ # First, get the list of bounces register against the site list. For
+ # these addresses, we want to register a bounce on every list the
+ # address is a member of -- which we don't know yet.
+ sitebounces = self._bounces.get(mm_cfg.MAILMAN_SITE_LIST)
+ if sitebounces:
+ listnames = Utils.list_names()
else:
+ listnames = self._bounces.keys()
+ for listname in listnames:
+ mlist = self._open_list(listname)
+ mlist.Lock()
try:
- mlist.Lock(timeout=mm_cfg.LIST_LOCK_TIMEOUT)
- except LockFile.TimeOutError:
- # Try again later
- syslog('bounce', "%s: couldn't get list lock", listname)
- return 1
- else:
- try:
- for addr in addrs:
- if mlist.isMember(addr):
- mlist.registerBounce(addr, msg)
- foundp = 1
- mlist.Save()
- finally:
- mlist.Unlock()
- if not foundp:
- # It means an address was recognized but it wasn't an address
- # that's on any mailing list at this site. BAW: don't forward
- # these, but do log it.
- syslog('bounce', 'bounce message with non-members of %s: %s',
- listname, COMMASPACE.join(addrs))
- #maybe_forward(mlist, msg)
+ events = self._bounces.get(listname, []) + sitebounces
+ for addr, day, msg in events:
+ mlist.registerBounce(addr, msg, day=day)
+ mlist.Save()
+ finally:
+ mlist.Unlock()
+ # Reset and free all the cached memory
+ self._bounces = {}
+ self._bouncecnt = 0
+
+ def _cleanup(self):
+ self._register_bounces()
+ Runner._cleanup(self)
_______________________________________________
Mailman-checkins mailing list
[EMAIL PROTECTED]
http://mail.python.org/mailman/listinfo/mailman-checkins
