Update of /cvsroot/mailman/mailman3/src/mailman
In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv31133/src/mailman
Added Files:
Tag: 3.0
__init__.py adapters.py dbmgr.py errors.py i18n.py lock.py
lockfile.py message.py mlist.py profile.py startup.py utils.py
workthread.py
Log Message:
Initial checkin of Mailman3 source
--- NEW FILE: __init__.py ---
--- NEW FILE: adapters.py ---
# Copyright (C) 2004 Barry A. Warsaw
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 2 of the License, or (at your option)
# any later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
# more details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 59 Temple
# Place - Suite 330, Boston, MA 02111-1307, USA.
from mailman.categories import allvars
from mailman.text.strings import dstring
class varhelpdict:
"""Dictionary which returns linked varhelp variables."""
def __init__(self, listname):
self._listname = listname
def __getitem__(self, key):
return '<a href="/varhelp/%s/%s">%s</a>' % (self._listname, key, key)
class AttributeWrapper(object):
def __init__(self, prop, mlist):
self._prop = prop
self._mlist = mlist
self._varhelpdict = varhelpdict(self._mlist.listname)
def value(self):
value = getattr(self._mlist, self._prop.name)
return self._prop.format(value)
def description(self):
return dstring(self._prop.description) % self._varhelpdict
def details(self):
return dstring(self._prop.details) % self._varhelpdict
def __getattr__(self, attrname):
return getattr(self._prop, attrname)
class MailListAdapter(object):
onmlist = ('listname', 'domain')
def __init__(self, mlist, category):
self._mlist = mlist
self._category = category
def __getattr__(self, attrname):
if attrname in self.onmlist:
return getattr(self._mlist, attrname)
prop, category = allvars.get(attrname)
if not prop:
raise AttributeError(attrname)
value = getattr(self._mlist, prop.name)
return prop.format(value)
def attributes(self):
for attr in self._category.attributes:
yield AttributeWrapper(attr, self._mlist)
def categorypath(self):
cat = self._category
if cat.parent:
return '%s/%s/%s' % (
self._mlist.listname, cat.parent.name, cat.name)
return '%s/%s' % (self._mlist.listname, cat.name)
--- NEW FILE: dbmgr.py ---
# Copyright (C) 2004 Barry A. Warsaw
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 2 of the License, or (at your option)
# any later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
# more details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 59 Temple
# Place - Suite 330, Boston, MA 02111-1307, USA.
"""Generic database manager."""
from mailman import errors
class DatabaseManager(object):
def __init__(self):
self._databases = []
self._lists = self._users = self._messages = self._members = None
def _setlists(self, db):
if self._lists is not None:
raise errors.ConfigurationError('List database already specified')
self._lists = db
self._add(db)
def _getlists(self):
return self._lists
lists = property(_getlists, _setlists)
def _setusers(self, db):
if self._users is not None:
raise errors.ConfigurationError('User database already specified')
self._users = db
self._add(db)
def _getusers(self):
return self._users
users = property(_getusers, _setusers)
def _setmembers(self, db):
if self._members is not None:
raise errors.ConfigurationError(
'Members database already specified')
self._members = db
self._add(db)
def _getmembers(self):
return self._members
members = property(_getmembers, _setmembers)
def _setmessages(self, db):
if self._messages is not None:
raise errors.ConfigurationError(
'Message database already specified')
self._messages = db
self._add(db)
def _getmessages(self):
return self._messages
messages = property(_getmessages, _setmessages)
def _add(self, db):
if db not in self._databases:
self._databases.append(db)
def shutdown(self):
for db in self._databases:
db.close()
dbmgr = DatabaseManager()
def startup():
dbmgr.lists.open()
dbmgr.users.open()
#dbmgr.messages.open()
shutdown = dbmgr.shutdown
--- NEW FILE: errors.py ---
# Copyright (C) 2004 Barry A. Warsaw
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 2 of the License, or (at your option)
# any later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
# more details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 59 Temple
# Place - Suite 330, Boston, MA 02111-1307, USA.
"""Exceptions for Mailman.
"""
# Base class for all Mailman exceptions
class MailmanError(Exception):
"""Generic Mailman exception."""
# List related errors
class MailingListError(MailmanError):
"""Base exception for list related errors."""
class NoListError(MailingListError):
"""The list does not exist."""
class ListExistsError(MailingListError):
"""An attempt was made to create an existing list."""
# User and membership errors
class UserError(MailmanError):
"""Exceptions associated with users."""
class NoUserError(UserError):
"""No user with the given id or address exists."""
class UserExistsError(UserError):
"""The user with the given id or address already exists."""
class UserAddressError(UserError):
"""There is already an address associated with the given label."""
class AddressError(MailmanError):
"""Base class for email address errors."""
class BadAddress(AddressError):
"""A bad email address."""
class NoValidDestination(AddressError):
"""A maildir-handled message had no valid destination."""
# Disposition errors are what handling modules raise when they want to
# short-circuit the pipeline processing. These aren't strictly exceptions,
# but it's convenient to model them as such.
class MessageDisposition(MailmanError):
"""Pipeline disposition of a message."""
class DiscardMessage(MessageDisposition):
"""Discard the message w/o a notification."""
class HoldMessage(MessageDisposition):
"""Hold a message for moderator approval."""
class RejectMessage(MessageDisposition):
"""Bounce a message."""
# Random other errors
class ConfigurationError(MailmanError):
"""Error in configurations."""
class DatabaseAlreadyOpen(MailmanError):
"""Database is already open."""
--- NEW FILE: i18n.py ---
# Copyright (C) 2004 Barry A. Warsaw
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 2 of the License, or (at your option)
# any later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
# more details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 59 Temple
# Place - Suite 330, Boston, MA 02111-1307, USA.
"""Internationalization support."""
from mailman.text.strings import istring
from mailman.text.safedict import nsdict
def _(s):
# Here's where we do i18n lookup
return istring(s) % nsdict()
--- NEW FILE: lock.py ---
# Copyright (C) 2004 Barry A. Warsaw
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 2 of the License, or (at your option)
# any later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
# more details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 59 Temple
# Place - Suite 330, Boston, MA 02111-1307, USA.
"""Interprocess locking.
"""
# Note that this implementation is not NFS-safe. It isn't clear that the
# Mailman 2.x NFS-safe implementation is necessary, but it definitely adds
# complexity.
import os
import time
import errno
import fcntl
import random
from mailman.errors import TimeOutError
class Lock:
def __init__(self, path):
self._path = path
self._islocked = False
try:
self._fp = open(path, 'r+')
except IOError, e:
if e.errno <> errno.ENOENT: raise
self._fp = open(path, 'w+')
def close(self):
if self._fp is not None:
self.unlock()
self._fp.close()
try:
os.unlink(self._path)
except OSError, e:
if e.errno <> errno.ENOENT:
raise
self._fp = None
def __del__(self):
self.close()
def isLocked(self):
return self._islocked
def lock(self, timeout=None):
"""Acquire the lock
With no timeout, a blocking lock is acquired. A timeout is a hint in
floating seconds for the length of time to attempt to acquire the
lock. If no lock can be acquired during that time, a TimeOutError is
raised.
"""
fileno = self._fp.fileno()
if self._islocked:
return
if timeout is None:
# block until we have the lock
fcntl.flock(fileno, fcntl.LOCK_EX)
self._islocked = True
return
expires = time.time() + timeout
while True:
try:
fcntl.flock(fileno, fcntl.LOCK_EX | fcntl.LOCK_NB)
self._islocked = True
break
except IOError, e:
if e.errno not in (errno.EAGAIN, errno.EACCES):
raise
if time.time() > expires:
raise TimeOutError
self._sleep()
def unlock(self):
if not self._islocked:
return
fcntl.flock(self._fp.fileno(), fcntl.LOCK_UN)
self._islocked = False
def _sleep(self):
interval = random.random() * 2.0 + 0.01
time.sleep(interval)
--- NEW FILE: lockfile.py ---
# Copyright (C) 2004 by Barry A. Warsaw
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
"""Portable, NFS-safe file locking with timeouts.
This code implements an NFS-safe file-based locking algorithm influenced by
the GNU/Linux open(2) manpage, under the description of the O_EXCL option.
>From RH6.1:
[...] O_EXCL is broken on NFS file systems, programs which rely on it
for performing locking tasks will contain a race condition. The
solution for performing atomic file locking using a lockfile is to
create a unique file on the same fs (e.g., incorporating hostname and
pid), use link(2) to make a link to the lockfile. If link() returns
0, the lock is successful. Otherwise, use stat(2) on the unique file
to check if its link count has increased to 2, in which case the lock
is also successful.
The assumption made here is that there will be no `outside interference',
e.g. no agent external to this code will have access to link() to the affected
lock files.
LockFile objects support lock-breaking.
Locks have a `lifetime', which is the maximum length of time the process
expects to retain the lock. It is important to pick a good number here
because other processes will not break an existing lock until the expected
lifetime has expired. Too long and other processes will hang; too short and
you'll end up trampling on existing process locks -- and possibly corrupting
data. In a distributed (NFS) environment, you also need to make sure that
your clocks are properly synchronized.
"""
import os
import time
import errno
import random
import socket
import logging
import datetime
DEFAULT_LOCK_LIFETIME = datetime.timedelta(seconds=15)
# Allowable a bit of clock skew
CLOCK_SLOP = datetime.timedelta(seconds=10)
log = logging.getLogger('locks')
# This object is returned from lock() if the lock was acquired after it was
# stolen from some other locker.
WasBroken = object()
# This object is returned from lock() if we found a stale lock, i.e. the lock
# was claimed by the pid of a process that didn't exist.
WasStale = object()
# Exceptions that can be raised by this module
class LockError(Exception):
"""Base class for all exceptions in this module."""
class AlreadyLockedError(LockError):
"""An attempt is made to lock an already locked object."""
class NotLockedError(LockError):
"""An attempt is made to unlock an object that isn't locked."""
class TimeOutError(LockError):
"""The timeout interval elapsed before the lock succeeded."""
class LockFile:
"""A portable way to lock resources by way of the file system.
This class supports the following methods:
__init__(lockfile[, lifetime]):
Create the resource lock using lockfile as the global lock file. Each
process laying claim to this resource lock will create their own
temporary lock files based on the path specified by lockfile.
Optional lifetime is the number of seconds (or a datetime.timedelta
object representing the number of seconds) the process expects to hold
the lock.
set_lifetime(lifetime):
Set a new lock lifetime. This takes affect the next time the file is
locked, but does not refresh a locked file.
get_lifetime():
Return the lock's lifetime.
refresh([newlifetime[, unconditionally]]):
Refreshes the lifetime of a locked file. Use this if you realize that
you need to keep a resource locked longer than you thought. With
optional newlifetime, set the lock's lifetime. Raises NotLockedError
if the lock is not set, unless optional unconditionally flag is set to
true.
lock([timeout]):
Acquire the lock. This blocks until the lock is acquired unless
optional timeout is greater than 0, in which case, a TimeOutError is
raised when timeout number of seconds (or possibly more) expires
without lock acquisition. Raises AlreadyLockedError if the lock is
already set.
unlock([unconditionally]):
Relinquishes the lock. Raises a NotLockedError if the lock is not
set, unless optional unconditionally is true.
locked():
Return true if the lock is set, otherwise false. To avoid race
conditions, this refreshes the lock (on set locks).
"""
# BAW: We need to watch out for two lock objects in the same process
# pointing to the same lock file. Without this, if you lock lf1 and do
# not lock lf2, lf2.locked() will still return true. NOTE: this gimmick
# probably does /not/ work in a multithreaded world, but we don't have to
# worry about that, do we? <1 wink>.
counter = 0
def __init__(self, lockfile, lifetime=None):
"""Create the resource lock using lockfile as the global lock file.
Each process laying claim to this resource lock will create their own
temporary lock files based on the path specified by lockfile.
Optional lifetime is the number of seconds the process expects to hold
the lock.
"""
self._lockfile = lockfile
self.set_lifetime(lifetime or DEFAULT_LOCK_LIFETIME)
# This works because we know we're single threaded
self._counter = LockFile.counter
LockFile.counter += 1
self._tmpfname = '%s.%s.%d.%d' % (
lockfile, socket.gethostname(), os.getpid(), self._counter)
self._logprefix = os.path.split(self._lockfile)[1]
# For transferring ownership across a fork.
self._owned = True
def __repr__(self):
return '<LockFile %s: %s [%s: %ssec] pid=%s>' % (
id(self), self._lockfile,
self.locked() and 'locked' or 'unlocked',
self._lifetime, os.getpid())
def set_lifetime(self, lifetime):
"""Set a new lock lifetime.
lifetime can be a number of seconds or a datetime.timedelta object.
This takes affect the next time the file is locked, but does not
refresh a locked file.
"""
try:
seconds = lifetime.seconds
except AttributeError:
seconds = lifetime
self._lifetime = datetime.timedelta(seconds=seconds)
def get_lifetime(self):
"""Return the lock's lifetime."""
return self._lifetime
def refresh(self, newlifetime=None, unconditionally=False):
"""Refreshes the lifetime of a locked file.
Use this if you realize that you need to keep a resource locked longer
than you thought. With optional newlifetime, set the lock's lifetime.
Raises NotLockedError if the lock is not set, unless optional
unconditionally flag is set to true.
"""
if newlifetime is not None:
self.set_lifetime(newlifetime)
# Do we have the lock? As a side effect, this refreshes the lock!
if not self.locked() and not unconditionally:
raise NotLockedError('%s: %s' % (repr(self), self._read()))
def lock(self, timeout=None):
"""Acquire the lock.
This blocks until the lock is acquired unless optional timeout is
given, in which case a TimeOutError is raised when timeout number of
seconds (or possibly more) expires without lock acquisition. timeout
must be a datetime.timedelta object.
Raises AlreadyLockedError if the lock is already set.
This method returns None, unless the lock was broken and stolen from
some other process, in which case, it returns WasBroken. If the lock
appeared to be owned by a stale process, then WasStale is returned.
"""
if timeout:
timeout_time = datetime.datetime.now() + timeout
rtnval = None
# Make sure my temp lockfile exists, and that its contents are
# up-to-date (e.g. the temp file name, and the lock lifetime).
self._write()
self._touch()
log.debug('laying claim')
# for quieting the logging output
loopcount = -1
while True:
loopcount += 1
# Create the hard link and test for exactly 2 links to the file
try:
os.link(self._tmpfname, self._lockfile)
# If we got here, we know we know we got the lock, and never
# had it before, so we're done. Just touch it again for the
# fun of it.
log.debug('got the lock')
self._touch()
break
except OSError, e:
# The link failed for some reason, possibly because someone
# else already has the lock (i.e. we got an EEXIST), or for
# some other bizarre reason.
if e.errno == errno.ENOENT:
# BAW: in some Linux environments, it is possible to get
# an ENOENT, which is truly strange, because this means
# that self._tmpfname doesn't exist at the time of the
# os.link(), but self._write() is supposed to guarantee
# that this happens! I don't honestly know why this
# happens, but for now we just say we didn't acquire the
# lock, and try again next time.
pass
elif e.errno <> errno.EEXIST:
# Something very bizarre happened. Clean up our state and
# pass the error on up.
log.error('unexpected link error: %s', e)
os.unlink(self._tmpfname)
raise
elif self._linkcount() <> 2:
# Somebody's messin' with us! Log this, and try again
# later. BAW: should we raise an exception?
log.error('unexpected linkcount: %d', self._linkcount())
elif self._read() == self._tmpfname:
# It was us that already had the link.
log.debug('already locked')
raise AlreadyLockedError
# Otherwise, someone else has the lock.
pass
# We did not acquire the lock, because someone else already has
# it. Have we timed out in our quest for the lock?
now = datetime.datetime.now()
if timeout and timeout_time < now:
os.unlink(self._tmpfname)
log.debug('timed out')
raise TimeOutError
# Let's see if that somebody else is still around.
owner = self._read()
host, pid, counter = owner.split('.')[-3:]
pid = int(pid)
if host == socket.gethostname():
try:
os.kill(pid, 0)
except OSError, e:
if e.errno <> errno.ESRCH: raise
# The process does not exist, so the lock is stale
self._break()
rtnval = WasStale
log.warning('stale lock broken: %s', self._lockfile)
# Okay, we haven't timed out, but we didn't get the lock. Let's
# find if the lock lifetime has expired.
if now > self._releasetime():
# Yes, so break the lock.
self._break()
rtnval = WasBroken
log.warning('breaking expired lock: %s', self._lockfile)
# Okay, someone else has the lock, our claim hasn't timed out yet,
# and the expected lock lifetime hasn't expired yet. So let's
# wait a while for the owner of the lock to give it up.
if not loopcount % 100:
log.debug('waiting for claim')
# Sleep for a while and try again.
interval = random.random() * 2.0 + 0.01
time.sleep(interval)
return rtnval
def unlock(self, unconditionally=False):
"""Unlock the lock.
If we don't already own the lock (either because of unbalanced unlock
calls, or because the lock was stolen out from under us), raise a
NotLockedError, unless optional `unconditionally' is true.
"""
islocked = self.locked()
if not islocked and not unconditionally:
raise NotLockedError
# If we owned the lock, remove the global file, relinquishing it.
if islocked:
try:
os.unlink(self._lockfile)
except OSError, e:
if e.errno <> errno.ENOENT: raise
# Remove our tempfile
try:
os.unlink(self._tmpfname)
except OSError, e:
if e.errno <> errno.ENOENT: raise
log.debug('unlocked')
def locked(self):
"""Return True if we own the lock, False if we do not.
Checking the status of the lock resets the lock's lifetime, which
helps avoid race conditions during the lock status test.
"""
# Discourage breaking the lock for a while.
try:
self._touch()
except OSError, e:
if e.errno <> errno.EPERM: raise
# We can't touch the file because we're not the owner. I
# don't see how we can own the lock if we're not the owner.
return False
# BAW: can the link count ever be > 2?
if self._linkcount() <> 2:
return False
return self._read() == self._tmpfname
def finalize(self):
self.unlock(unconditionally=True)
# Use these only if you're transfering ownership to a child process across
# a fork. Use at your own risk, but it should be race-condition safe.
# _transfer_to() is called in the parent, passing in the pid of the child.
# _take_possession() is called in the child, and blocks until the parent
# has transferred possession to the child. _disown() is used to set the
# _owned flag to False, and it is a disgusting wart necessary to make
# forced lock acquisition work.
def _transfer_to(self, pid):
# First touch it so it won't get broken while we're fiddling about.
self._touch()
# Find out current claim's temp filename
winner = self._read()
# Now twiddle ours to the given pid
self._tmpfname = '%s.%s.%d' % (
self._lockfile, socket.gethostname(), pid)
# Create a hard link from the global lock file to the temp file. This
# actually does things in reverse order of normal operation because we
# know that lockfile exists, and tmpfname better not!
os.link(self._lockfile, self._tmpfname)
# Now update the lock file to contain a reference to the new owner
self._write()
# Toggle off our ownership of the file so we don't try to finalize it
# in our __del__()
self._owned = False
# Unlink the old winner, completing the transfer
os.unlink(winner)
# And do some sanity checks
assert self._linkcount() == 2
assert self.locked()
log.debug('transferred the lock')
def _take_possession(self):
self._tmpfname = tmpfname = '%s.%s.%d' % (
self._lockfile, socket.gethostname(), os.getpid())
# Wait until the linkcount is 2, indicating the parent has completed
# the transfer.
while self._linkcount() <> 2 or self._read() <> tmpfname:
time.sleep(0.25)
log.debug('took possession of the lock')
def _disown(self):
self._owned = False
#
# Private interface
#
def _write(self):
# Make sure it's group writable
oldmask = os.umask(002)
try:
fp = open(self._tmpfname, 'w')
fp.write(self._tmpfname)
fp.close()
finally:
os.umask(oldmask)
def _read(self):
try:
fp = open(self._lockfile)
filename = fp.read()
fp.close()
return filename
except EnvironmentError, e:
if e.errno <> errno.ENOENT: raise
return None
def _touch(self, filename=None):
future = datetime.datetime.now() + self._lifetime
t = time.mktime(future.timetuple())
try:
# TBD: We probably don't need to modify atime, but this is easier.
os.utime(filename or self._tmpfname, (t, t))
except OSError, e:
if e.errno <> errno.ENOENT: raise
def _releasetime(self):
try:
mtime = os.stat(self._lockfile).st_mtime
return datetime.datetime.fromtimestamp(mtime) + CLOCK_SLOP
except OSError, e:
if e.errno <> errno.ENOENT: raise
# now() can never be greater than this point in time
return datetime.datetime.max
def _linkcount(self):
try:
return os.stat(self._lockfile).st_nlink
except OSError, e:
if e.errno <> errno.ENOENT: raise
return -1
def _break(self):
# First, touch the global lock file. This reduces but does not
# eliminate the chance for a race condition during breaking. Two
# processes could both pass the test for lock expiry in lock() before
# one of them gets to touch the global lockfile. This shouldn't be
# too bad because all they'll do in this function is wax the lock
# files, not claim the lock, and we can be defensive for ENOENTs here.
#
# Touching the lock could fail if the process breaking the lock and
# the process that claimed the lock have different owners. We could
# solve this by set-uid'ing, but that's ugly.
try:
self._touch(self._lockfile)
except OSError, e:
if e.errno <> errno.EPERM: raise
# Get the name of the old winner's temp file.
winner = self._read()
# Remove the global lockfile, which actually breaks the lock.
try:
os.unlink(self._lockfile)
except OSError, e:
if e.errno <> errno.ENOENT: raise
# Try to remove the old winner's temp file, since we're assuming the
# winner process has hung or died. Don't worry too much if we can't
# unlink their temp file -- this doesn't wreck the locking algorithm,
# but will leave temp file turds laying around, a minor inconvenience.
try:
if winner:
os.unlink(winner)
except OSError, e:
if e.errno <> errno.ENOENT: raise
# Multi-process stress tests
def _dochild(childno):
prefix = '[%d]' % childno
# Create somewhere between 1 and 1000 locks
lockfile = LockFile('/tmp/LockTest', lifetime=120)
# Use a lock lifetime of between 1 and 15 seconds. Under normal
# situations, Mailman's usage patterns (untested) shouldn't be much longer
# than this.
workinterval = 5 * random.random()
hitwait = 20 * random.random()
print prefix, 'workinterval:', workinterval
islocked = False
t0 = 0
t1 = 0
t2 = 0
try:
try:
t0 = time.time()
print prefix, 'acquiring...'
lockfile.lock()
print prefix, 'acquired...'
islocked = True
except TimeOutError:
print prefix, 'timed out'
else:
t1 = time.time()
print prefix, 'acquisition time:', t1-t0, 'sec'
time.sleep(workinterval)
finally:
if islocked:
try:
lockfile.unlock()
t2 = time.time()
print prefix, 'released; lock hold time:', t2-t1, 'secs'
except NotLockedError:
print prefix, 'lock was broken'
# wait for next web hit
print prefix, 'sleep:', hitwait
time.sleep(hitwait)
def _seed():
try:
fp = open('/dev/random')
d = fp.read(40)
fp.close()
except EnvironmentError, e:
if e.errno <> errno.ENOENT:
raise
d = sha.new(`os.getpid()`+`time.time()`).hexdigest()
random.seed(d)
def _onetest(childno):
loopcount = random.randint(1, 100)
for i in range(loopcount):
print '[%d] Loop %d of %d' % (childno, i+1, loopcount)
pid = os.fork()
if pid:
# parent, wait for child to exit
pid, status = os.waitpid(pid, 0)
else:
# child
_seed()
try:
_dochild(childno)
except KeyboardInterrupt:
pass
os._exit(0)
def _reap(kids):
if not kids:
return
pid, status = os.waitpid(-1, os.WNOHANG)
if pid <> 0:
del kids[pid]
def _test(numtests):
kids = {}
for i in range(numtests):
pid = os.fork()
if pid:
# parent
kids[pid] = pid
else:
# child
_seed()
try:
_onetest(i)
except KeyboardInterrupt:
pass
os._exit(0)
# slightly randomize each kid's seed
while kids:
_reap(kids)
if __name__ == '__main__':
import sha
import sys
_test(int(sys.argv[1]))
--- NEW FILE: message.py ---
# Copyright (C) 2004 Barry A. Warsaw
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 2 of the License, or (at your option)
# any later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
# more details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 59 Temple
# Place - Suite 330, Boston, MA 02111-1307, USA.
"""Standard Mailman message object.
This is a subclass of mimeo.Message but provides a slightly extended interface
which is more convenient for use inside Mailman.
"""
import re
import email
import email.Message
import email.Utils
from email.Charset import Charset
from email.Header import Header
from mailman import config
COMMASPACE = ', '
class Message(email.Message.Message):
# BAW: For debugging w/ bin/dumpdb. Apparently pprint uses repr.
def __repr__(self):
return self.__str__()
# I think this method ought to eventually be deprecated
def get_sender(self, use_envelope=None, preserve_case=False):
"""Return the address considered to be the author of the email.
This can return either the From: header, the Sender: header or the
envelope header (a.k.a. the unixfrom header). The first non-empty
header value found is returned. However the search order is
determined by the following:
- If mm_cfg.USE_ENVELOPE_SENDER is true, then the search order is
Sender:, From:, unixfrom
- Otherwise, the search order is From:, Sender:, unixfrom
The optional argument use_envelope, if given overrides the
mm_cfg.USE_ENVELOPE_SENDER setting. It should be set to either 0 or 1
(don't use None since that indicates no-override).
unixfrom should never be empty. The return address is always
lowercased, unless preserve_case is true.
This method differs from get_senders() in that it returns one and only
one address, and uses a different search order.
"""
senderfirst = mm_cfg.USE_ENVELOPE_SENDER
if use_envelope is not None:
senderfirst = use_envelope
if senderfirst:
headers = ('sender', 'from')
else:
headers = ('from', 'sender')
for h in headers:
# Use only the first occurrance of Sender: or From:, although it's
# not likely there will be more than one.
fieldval = self[h]
if not fieldval:
continue
addrs = email.Utils.getaddresses([fieldval])
try:
realname, address = addrs[0]
except IndexError:
continue
if address:
break
else:
# We didn't find a non-empty header, so let's fall back to the
# unixfrom address. This should never be empty, but if it ever
# is, it's probably a Really Bad Thing. Further, we just assume
# that if the unixfrom exists, the second field is the address.
unixfrom = self.get_unixfrom()
if unixfrom:
address = unixfrom.split()[1]
else:
# TBD: now what?!
address = ''
if not preserve_case:
return address.lower()
return address
def get_senders(self, preserve_case=False, headers=None):
"""Return a list of addresses representing the author of the email.
The list will contain the following addresses (in order)
depending on availability:
1. From:
2. unixfrom
3. Reply-To:
4. Sender:
The return addresses are always lower cased, unless `preserve_case' is
true. Optional `headers' gives an alternative search order, with None
meaning, search the unixfrom header. Items in `headers' are field
names without the trailing colon.
"""
if headers is None:
headers = ('from', None, 'reply-to', 'sender')
pairs = []
for h in headers:
if h is None:
# get_unixfrom() returns None if there's no envelope
fieldval = self.get_unixfrom() or ''
try:
pairs.append(('', fieldval.split()[1]))
except IndexError:
# Ignore badly formatted unixfroms
pass
else:
fieldvals = self.get_all(h)
if fieldvals:
pairs.extend(email.Utils.getaddresses(fieldvals))
authors = []
for pair in pairs:
address = pair[1]
if address is not None and not preserve_case:
address = address.lower()
authors.append(address)
return authors
class UserNotification(Message):
"""Class for internally crafted messages."""
def __init__(self, recip, sender, subject=None, text=None, lang=None):
Message.__init__(self)
charset = None
if lang is not None:
charset = Charset(Utils.GetCharSet(lang))
if text is not None:
self.set_payload(text, charset)
if subject is None:
subject = '(no subject)'
self['Subject'] = Header(subject, charset, header_name='Subject',
errors='replace')
self['From'] = sender
if isinstance(recip, list):
self['To'] = COMMASPACE.join(recip)
self.recips = recip
else:
self['To'] = recip
self.recips = [recip]
def send(self, mlist, **_kws):
"""Sends the message by enqueuing it to the `virgin' queue.
This is used for all internally crafted messages.
"""
# Since we're crafting the message from whole cloth, let's make sure
# this message has a Message-ID. Yes, the MTA would give us one, but
# this is useful for logging to logs/smtp.
if not self.has_key('message-id'):
self['Message-ID'] = Utils.unique_message_id(mlist)
# Ditto for Date: which is required by RFC 2822
if not self.has_key('date'):
self['Date'] = email.Utils.formatdate(localtime=True)
# UserNotifications are typically for admin messages, and for messages
# other than list explosions. Send these out as Precedence: bulk, but
# don't override an existing Precedence: header.
if not self.has_key('precedence'):
self['Precedence'] = 'bulk'
self._enqueue(mlist, **_kws)
def _enqueue(self, mlist, **_kws):
# Not imported at module scope to avoid import loop
from mailman.queues.switchboard import factory
virginq = factory.get(config.virginqdir)
# The message metadata better have a `recip' attribute
virginq.enqueue(self,
listname = mlist.internal_name(),
recips = self.recips,
nodecorate = True,
reduced_list_headers = True,
**_kws)
class OwnerNotification(UserNotification):
"""Like user notifications, but this message goes to the list owners."""
def __init__(self, mlist, subject=None, text=None, tomoderators=True):
recips = mlist.owner[:]
if tomoderators:
recips.extend(mlist.moderator)
# We have to set the owner to the site's -bounces address, otherwise
# we'll get a mail loop if an owner's address bounces.
sender = Utils.get_site_email(mlist.host_name, 'bounces')
lang = mlist.preferred_language
UserNotification.__init__(self, recips, sender, subject, text, lang)
# Hack the To header to look like it's going to the -owner address
del self['to']
self['To'] = mlist.GetOwnerEmail()
self._sender = sender
def _enqueue(self, mlist, **_kws):
# Not imported at module scope to avoid import loop
from Mailman.Queue.sbcache import get_switchboard
virginq = get_switchboard(mm_cfg.VIRGINQUEUE_DIR)
# The message metadata better have a `recip' attribute
virginq.enqueue(self,
listname = mlist.internal_name(),
recips = self.recips,
nodecorate = True,
reduced_list_headers = True,
envsender = self._sender,
**_kws)
--- NEW FILE: mlist.py ---
# Copyright (C) 2004 Barry A. Warsaw
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 2 of the License, or (at your option)
# any later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
# more details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 59 Temple
# Place - Suite 330, Boston, MA 02111-1307, USA.
import sys
import logging
from mailman import categories
from mailman import errors
from mailman import utils
from mailman.config import configuration
from mailman.dbmgr import dbmgr
log = logging.getLogger('mailman.events')
class MailList(object):
def open(listname, domainName):
"""An alternative interface to construct MailList objects."""
listid = '[EMAIL PROTECTED]' % (listname, domainName)
return MailList(listid)
open = staticmethod(open)
def create(listname, domain, style='default'):
listname = listname.lower()
# Verify that the listname is valid by checking to make sure that the
# posting address is valid.
mailhost, webhost = configuration.domains.bydomain[domain]
postaddr = '[EMAIL PROTECTED]' % (listname, mailhost)
if not utils.is_valid_address(postaddr):
raise errors.MailingListError(
'Bad list posting address: %s' % postaddr)
listid = listname + '@' + domain
utcnow = dbmgr.lists.create_list(listid)
log.info('List %s created at %s UTC', listid, utcnow)
mlist = MailList(listid)
# Basic variable initialization
for name, (prop, cat) in categories.allvars.items():
setattr(mlist, name, prop.default())
# Populate it with attributes only the create() function knows.
mlist.listid = listid
mlist.listname = listname
mlist.mailhost = mailhost
mlist.webhost = webhost
mlist.created = utcnow
# Apply style initializations.
stylemod = 'mailman.styles.' + style
__import__(stylemod)
mod = sys.modules[stylemod]
mod.initialize(mlist)
return mlist
create = staticmethod(create)
def __init__(self, listid):
if not dbmgr.lists.list_exists(listid):
raise errors.NoListError(listid)
self._listid = listid
def __getattribute__(self, attr):
if attr.startswith('_'):
return super(MailList, self).__getattribute__(attr)
return dbmgr.lists.get(self._listid, attr)
def __setattr__(self, attr, val):
if attr.startswith('_'):
super(MailList, self).__setattr__(attr, val)
return
dbmgr.lists.set(self._listid, attr, val)
--- NEW FILE: profile.py ---
# Copyright (C) 2004 Barry A. Warsaw
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 2 of the License, or (at your option)
# any later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
# more details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 59 Temple
# Place - Suite 330, Boston, MA 02111-1307, USA.
"""User profiles."""
from mailman import errors
from mailman import utils
from mailman.dbmgr import dbmgr
from mailman.interfaces.profiles import IProfile
class Profile(object):
__implements__ = IProfile
def __init__(self, userid):
self._userid = userid
def __str__(self):
return self.address
def __repr__(self):
return '<Profile for %s [%r]>' % (self.address, self._userid)
# IProfile
def _getUserid(self):
return self._userid
userid = property(_getUserid)
def _getAddress(self):
return dbmgr.users.getAddress(self._userid)
def _setAddress(self, address):
dbmgr.users.setAddress(self._userid, address)
address = property(_getAddress, _setAddress)
def _getRealname(self):
return dbmgr.users.getRealname(self._userid)
def _setRealname(self, name):
dbmgr.users.setRealname(self._userid, name)
realname = property(_getRealname, _setRealname)
def _getPassword(self):
return dbmgr.users.getPasswordHash(self._userid)
def _setPassword(self, password):
dbmgr.users.setPasswordHash(self._userid, password)
password = property(_getPassword, _setPassword)
def _getAliases(self):
return dbmgr.users.getAliases(self._id)
def _setAliases(self, aliases):
dbmgr.users.setAliases(self._id, aliases)
aliases = property(_getAliases, _setAliases)
def _getDefaults(self):
return BDBPreferences(dbmgr.users, self._id, '')
defaults = property(_getDefaults)
def _getStatus(self):
return BDBDeliveryStatus(dbmgr.users, self._id, '')
status = property(_getStatus)
def _getBounceInfo(self):
return BDBBounceInfo(dbmgr.users, self._id)
bounceInfo = property(_getBounceInfo)
def _getListsOn(self):
return dbmgr.users._getListsOn(self._id)
def _setListsOn(self, listids):
dbmgr.users.setListsOn(self._id, listids)
listsOn = property(_getListsOn, _setListsOn)
# XXX roles
def getOrCreateProfile(address):
"""Get the profile for address or create one if necessary.
Returns an IProfile and a flag specifying whether the profile was newly
created or not.
"""
userid = dbmgr.users.useridForAddress(address)
if userid:
return dbmgr.users.getProfile(userid), False
# Verify that the owner email address is legal
if not utils.isValidAddress(address):
raise errors.UserError('Bad email address: %s' % address)
return dbmgr.users.createProfile(address), True
--- NEW FILE: startup.py ---
# Copyright (C) 2004 Barry A. Warsaw
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 2 of the License, or (at your option)
# any later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
# more details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 59 Temple
# Place - Suite 330, Boston, MA 02111-1307, USA.
import sys
import atexit
from mailman.config import configuration
_shutdowns = []
def startup():
# Only startup once
if _shutdowns:
return
for m in configuration.startups.component:
pkgname = 'mailman.%s' % m
__import__(pkgname)
pkg = sys.modules[pkgname]
pkg.startup()
_shutdowns.append(pkg)
# Shut things down in reverse order
_shutdowns.reverse()
def shutdown():
for m in _shutdowns:
m.shutdown()
atexit.register(shutdown)
--- NEW FILE: utils.py ---
# Copyright (C) 2004 Barry A. Warsaw
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 2 of the License, or (at your option)
# any later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
# more details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 59 Temple
# Place - Suite 330, Boston, MA 02111-1307, USA.
"""Various and sundry utilities for Mailman."""
import os
import re
import sha
import errno
import random
from datetime import timedelta
EMPTYSTRING = ''
def seconds(s): return s
def minutes(m): return m * 60
def hours(h): return minutes(h) * 60
def days(d): return hours(d) * 24
def _syllable():
__pychecker__ = 'no-stringiter'
# Funky algorithm for creating random, but word-like passwords
syls = []
for v in 'aeiou':
for c in 'bcdfghkmnprstvwxz':
syls.append(c + v)
syls.append(v + c)
while True:
yield random.choice(syls)
syllable = _syllable()
def mksyllables(length=6):
__pychecker__ = 'unusednames=i'
return EMPTYSTRING.join([syllable.next() for i in range(length // 2)])
# Wait on a dictionary of child pids
def reap(kids, func=None, once=False):
while kids:
if func:
func()
try:
pid, status = os.waitpid(-1, os.WNOHANG)
except OSError, e:
# If the child procs had a bug we might have no children
if e.errno <> errno.ECHILD:
raise
kids.clear()
break
if pid <> 0:
try:
del kids[pid]
except KeyError:
# Huh? How can this happen?
pass
if once:
break
# XXX: We should validate completely based on RFC 2822, and perhaps even do
# active validation by contacting the MTA in the hostname part.
# TBD: what other characters should be disallowed?
_badchars = re.compile(r'[][()<>|;^,/\200-\377\s]')
def is_valid_address(addr):
if not addr:
return False
if _badchars.search(addr) or addr[0] == '-':
return False
localpart, domain = addr.split('@', 1)
if '@' in domain:
return False
if not '.' in domain:
return False
return True
def hash_password(password):
return sha.new(password).digest()
--- NEW FILE: workthread.py ---
# Copyright (C) 2004 Barry A. Warsaw
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 2 of the License, or (at your option)
# any later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
# more details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 59 Temple
# Place - Suite 330, Boston, MA 02111-1307, USA.
import datetime
import threading
DEFAULT_TIMEOUT = 5.0 # seconds
class WorkThread(threading.Thread):
def __init__(self, event, log, timeout=DEFAULT_TIMEOUT, daemon=True):
threading.Thread.__init__(self)
self._event = event
self._log = log
self._interval = datetime.timedelta(seconds=timeout)
self._done = False
self._nextcheck = datetime.datetime.now() + self._interval
self.setDaemon(daemon)
def run(self):
me = self.__class__.__name__
self._log.debug('%s thread started', me)
while not self._done:
if datetime.datetime.now() > self._nextcheck:
self._log.debug('%s thread working', me)
self._work()
# Recalculate now() because we don't know how long the work
# may have taken.
self._nextcheck = datetime.datetime.now() + self._interval
# Block w/ timeout on the stop event.
self._event.wait(self._interval.seconds)
self._done = self._event.isSet()
self._log.debug('%s thread finished', me)
def _work(self):
raise NotImplementedError
_______________________________________________
Mailman-checkins mailing list
[EMAIL PROTECTED]
Unsubscribe: http://mail.python.org/mailman/options/mailman-checkins/archive%40jab.org
