Update of /cvsroot/mailman/mailman3/src/mailman/smtp
In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv31133/src/mailman/smtp
Added Files:
Tag: 3.0
__init__.py bounceproc.py chunkify.py connection.py
decorate.py pool.py postfix.py verp.py
Log Message:
Initial checkin of Mailman3 source
--- NEW FILE: __init__.py ---
# Copyright (C) 2004 Barry A. Warsaw
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 2 of the License, or (at your option)
# any later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
# more details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 59 Temple
# Place - Suite 330, Boston, MA 02111-1307, USA.
from mailman.smtp.pool import ConnectionPool
from mailman.smtp.verp import VERPMailer
#from mailman.smtp.bulk import BulkMailer
_pool = None
_verp = None
_bulk = None
def send(mlist, msg, msgdata):
# Decide whether to bulk send or verp send this message
if msgdata.get('personalize') or mlist.personalize:
_verp.send(mlist, msg, msgdata)
else:
_bulk.send(mlist, msg, msgdata)
def startup():
global _pool, _verp, _bulk
_pool = ConnectionPool()
_pool.start()
_verp = VERPMailer(_pool)
# _bulk = BulkMailer(_pool)
def shutdown():
_pool.close()
--- NEW FILE: bounceproc.py ---
# Copyright (C) 2004 Barry A. Warsaw
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 2 of the License, or (at your option)
# any later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
# more details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 59 Temple
# Place - Suite 330, Boston, MA 02111-1307, USA.
import os
import sets
import errno
import logging
import datetime
import threading
from mailman.config import configuration
from mailman.workthread import WorkThread
log = logging.getLogger('mailman.bounces')
class BounceProc(object):
def __init__(self):
self._bouncefp = None
self._seqno = 0
self._count = 0
self._bouncefp = None
self._rlock = threading.RLock()
self._stop = threading.Event()
self._procthread = BounceProcessorThread(self._stop, self)
self._procthread.start()
def _open(self):
filename = 'bounces.%05d.%05d.open' % (os.getpid(), self._seqno)
self._seqno += 1
path = os.path.join(configuration.dirs.bouncedata, filename)
self._bouncefp = open(path, 'w')
self._count = 0
def close(self):
self._close()
self._stop.set()
def _close(self):
if self._bouncefp is None:
return
# Closing should flip the events file to .closed so that the
# processing thread will process the bounces, the next time its
# started up.
self._rlock.acquire()
try:
self._bouncefp.flush()
self._bouncefp.close()
oldname = self._bouncefp.name
newname = os.path.splitext(oldname)[0] + '.closed'
os.rename(oldname, newname)
finally:
self._rlock.release()
return newname
def register(self, addresses, date=None):
if self._bouncefp is None:
self._open()
if date is None:
date = datetime.date.now()
self._rlock.acquire()
try:
for address in addresses:
print >> self._bouncefp, address, date.isoformat()
self._count += 1
finally:
self._rlock.release()
def flip(self):
self._rlock.acquire()
try:
if not self._count:
return None
newname = self.close()
finally:
self._rlock.release()
self._open()
return newname
class BounceProcessorThread(WorkThread):
def __init__(self, event, bounceEvents):
WorkThread.__init__(self, event, log)
self._bounceEvents = bounceEvents
self._files = []
# First scan the event dir for existing .close files, and add them to
# the queue for processing.
for f in os.listdir(configuration.dirs.bouncedata):
path = os.path.join(configuration.dirs.bouncedata, f)
if path.endswith('.closed'):
self._files.append(path)
def _takefile(self, file):
if file is None:
return None
newfile = os.path.splitext(file)[0] + '.taken'
try:
os.rename(file, newfile)
return file
except OSError, e:
if e.errno <> errno.ENOENT: raise
return None
def _work(self):
# Process one file at a time. If self._files is empty, flip the event
# log to get another file to work on.
file = None
while self._files and not file:
file = self._takefile(self._files.pop())
if file is None:
file = self._takefile(self._bounceEvents.flip())
if file is None:
# Nothing to do
return
# Now process all the events in the file. We only process one bounce
# per address per day.
s = sets.Set()
fp = open(file)
try:
for line in fp:
address, date = line.split()
date = datetime.date(*[int(x) for x in date.split('-')])
if (address, date) in s:
# We already registered a bounce for this combination
continue
else:
#db.register(address, date)
log.debug('Bounce registered: %s at %s', address, date)
s.add((address, date))
finally:
fp.close()
--- NEW FILE: chunkify.py ---
# Copyright (C) 2004 Barry A. Warsaw
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 2 of the License, or (at your option)
# any later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
# more details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 59 Temple
# Place - Suite 330, Boston, MA 02111-1307, USA.
"""Chunkify a list of recipients.
The algorithm is as follows:
- sort the addresses by reverse domain
- fill those into max_recips buckets
I can imagine all manner of other algorithms for chunking, but we're
essentially playing a guessing game as to what's most efficient for the MTA
we're talking to. WIBNI we didn't even have to do this?
"""
from mailman.config import configuration
def chunkify(recips):
recipchunks = []
chunk = []
chunksize = 0
revrecips = sort(recips)
while revrecips:
chunk.append(revrecips.pop())
chunksize += 1
if chunksize == configuration.smtp.maxrecips:
recipchunks.append(chunk)
chunk = []
chunksize = 0
if chunk:
recipchunks.append(chunk)
return recipchunks
def sort(recips):
revrecips = []
for address in recips:
localpart, domain = address.split('@', 1)
parts = domain.split('.')
parts.reverse()
revrecips.append((tuple(parts), address))
revrecips.sort()
for parts, address in revrecips:
yield address
--- NEW FILE: connection.py ---
# Copyright (C) 2004 Barry A. Warsaw
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 2 of the License, or (at your option)
# any later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
# more details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 59 Temple
# Place - Suite 330, Boston, MA 02111-1307, USA.
"""Connection objects."""
import smtplib
class Connection(object):
"""The representation of one re-connectable connection."""
def __init__(self, host, port, maxsessions):
self._host = host
self._port = port
self._maxsessions = maxsessions
self._numsessions = None
self._conn = None
def _connect(self):
self._conn = smtplib.SMTP()
self._conn.connect(self._host, self._port)
self._numsessions = self._maxsessions
def send(self, envsender, recips, msgtext):
if self._conn is None:
self._connect()
try:
results = self._conn.sendmail(envsender, recips, msgtext)
except Exception:
# For safety, close this connection. The next send attempt will
# automatically re-open it. Pass the exception on up.
self.close()
raise
# This session has been successfully completed.
if self._numsessions is not None:
self._numsessions -= 1
if self._numsessions <= 0:
self.close()
return results
def close(self):
# Only quit the underlying connection if it actually is connected
if self._conn:
try:
self._conn.quit()
except smtplib.SMTPException:
pass
self._conn = None
--- NEW FILE: decorate.py ---
# Copyright (C) 2004 Barry A. Warsaw
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 2 of the License, or (at your option)
# any later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
# more details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 59 Temple
# Place - Suite 330, Boston, MA 02111-1307, USA.
"""Decorate the message with the appropriate header and footers."""
LATIN_1_15 = ('iso-8859-1', 'iso-8859-15')
def compatible(mcset, lcset):
# If the list's preferred charset is us-ascii, we can always safely add
# the header/footer to a plain text message since all email charsets
# Mailman supports are strict supersets of us-ascii -- no, UTF-16 emails
# are not supported.
if lcset == 'us-ascii':
return True
if mcset == lcset:
return True
# Latin 1 and Latin 9 are basically compatible
if lcset in LATIN_1_15 and mcset in LATIN_1_15:
return True
# For now, nothing else is compatible.
return False
def decorate(mlist, msg, msgdata):
if msgdata.get('nodecorate'):
return
if msgdata.get('isdigest') and not mlist.shared_decoration:
header = mlist.digest_header
footer = mlist.digest_footer
else:
header = mlist.msg_header
header = mlist.msg_footer
# Escape hatch if there's nothing to do
if not header and not footer:
return
# Attach the headers and footers in a MIME-compliant way.
#
# If the messages's content type is text/plain and the character sets are
# compatible, we can add these parts by concatenation to the payload.
mcset = msg.get_param('charset', 'us-ascii').lower()
# XXX get the list's preferred character set
lcset = 'us-ascii'
msgtype = msg.get_type('text/plain')
if not msg.is_multipart() and msgtype == 'text/plain' and \
msg.get('content-transfer-coding', '').lower() <> 'base64' and \
compatible(mcset, lcset):
payload = msg.get_payload()
frontsep = endsep = ''
if header and not header.endswith('\n'):
frontsep = '\n'
if footer and not payload.endswith('\n'):
endsep = '\n'
payload = header + frontsep + payload + endsep + footer
msg.set_payload(payload)
return
# For now, we don't add the headers and footers for any other combination.
--- NEW FILE: pool.py ---
# Copyright (C) 2004 Barry A. Warsaw
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 2 of the License, or (at your option)
# any later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
# more details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 59 Temple
# Place - Suite 330, Boston, MA 02111-1307, USA.
"""Connection pools.
You can specify any number of outgoing smtp servers that Mailman can send
messages out on, and the number of simultaneous connections per server.
Mailman will maintain a pool of connection objects per server and re-use these
for among the delivery threads that you can also specify.
"""
import os
import socket
import smtplib
import logging
import datetime
import threading
from Queue import Queue
from mailman.config import configuration
from mailman.smtp.bounceproc import BounceProc
from mailman.smtp.connection import Connection
from mailman.queues.switchboard import Switchboard
log = logging.getLogger('mailman.smtp')
class ConnectionPool(object):
def __init__(self):
__pychecker__ = 'unusednames=i'
self._queue = Queue()
# Create a connection objects, one for every connection we're allowed
# to make to each server. They'll each block on a job queue, which
# we'll fill for each message to be sent.
self._threads = []
for host, port, conns, sessions in configuration.smtp.server:
for i in range(conns):
conn = Connection(host, port, sessions)
t = ConnectionThread(self._queue, conn)
self._threads.append(t)
def start(self):
for t in self._threads:
t.start()
def close(self):
# Send the "end job" marker to all threads, then join them
for t in self._threads:
self._queue.put(None)
for t in self._threads:
t.join()
def send(self, envsender, recips, text, msg, msgdata):
self._queue.put((envsender, recips, text, msg, msgdata))
class ConnectionThread(threading.Thread):
def __init__(self, queue, conn):
threading.Thread.__init__(self)
self._queue = queue
self._conn = conn
retrydir = os.path.join(configuration.dirs.qdir, 'retry')
self._retryq = Switchboard(retrydir)
self._bounces = BounceProc()
def run(self):
while True:
# This will block until there is work to do
job = self._queue.get()
if job is None:
# We're shutting down
break
envsender, recips, text, msg, msgdata = job
try:
refused = self._conn.send(envsender, recips, text)
except smtplib.SMTPRecipientsRefused, e:
log.error('SMTPRecipientRefused: %s', msg['message-id'])
refused = e.recipients
log.info('refused recipients: %s', refused)
except (socket.error, smtplib.SMTPException, IOError), e:
log.error('other mta error: %s', e)
# If the exception had an associated error code, use it,
# otherwise, fake it with a non-triggering exception code
refused = {}
errcode = getattr(e, 'smtp_code', -1)
errmsg = getattr(e, 'smtp_error', 'ignore')
for r in recips:
refused[r] = (errcode, errmsg)
# Now go through all the refused recipients. Log permanent
# failures in the bounce events file. Collect temporary failures
# so that the message can be retried.
tempfailures = []
for recip, (code, smtpmsg) in refused.items():
# DRUMS is an internet draft, but it says:
#
# [RFC-821] incorrectly listed the error where an SMTP
# server exhausts its implementation limit on the number of
# RCPT commands ("too many recipients") as having reply
# code 552. The correct reply code for this condition is
# 452. Clients SHOULD treat a 552 code in this case as a
# temporary, rather than permanent failure so the logic
# below works.
#
if code >= 500 and code <> 552:
# A permanent failure
self._bounces.register(recip)
else:
# Deal with persistent transient failures by queuing them
# up for retry later.
tempfailures.append(recip)
# If there are any temporary failures, retry this message
if tempfailures:
self._retryq.enqueue(msg, msgdata,
recips=tempfailures,
last_recipcnt=len(recips),
last_attempt=datetime.datetime.now())
--- NEW FILE: postfix.py ---
# Copyright (C) 2004 Barry A Warsaw
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 2 of the License, or (at your option)
# any later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
# more details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 59 Temple
# Place - Suite 330, Boston, MA 02111-1307, USA.
"""Postfix-specific support."""
import os
from mailman.config import configuration
from mailman.database import dbmgr
def genaliases(fp):
print >> fp, """\
# This file is generated by Mailman, and is kept in sync with the binary hash
# file virtual-mailman.db. YOU SHOULD NOT MANUALLY EDIT THIS FILE unless you
# know what you're doing, and can keep the two files properly in sync. If you
# screw it up, you're on your own.
#
# Note that you should already have this virtual domain set up properly in
# your Postfix installation. See README.POSTFIX for details.
"""
domains = dbmgr.lists.domains()
if not domains:
return
print >> fp, '# VIRTUAL DOMAINS START'
for domain in domains:
print >> fp, domain, '\tIGNORE'
print >> fp, '# VIRTUAL DOMAINS END'
print >> fp
print >> fp, '# LOOP ADDRESSES START'
for domain in domains:
print >> fp, '[EMAIL PROTECTED]' % (
configuration.smtp.loop_address,
domain,
os.path.join(configuration.dirs.vardir,
configuration.smtp.loop_address + '.txt'))
print >> fp, '# LOOP ADDRESSES END'
print >> fp
for listid in dbmgr.lists.listIds():
listname, domain = listid.split('.', 1)
# The trailing slash tells Postfix this is a maildir alias
maildir = os.path.join(configuration.dirs.maildir, listid) + '/'
print >> fp, '# STANZA START:', listid
print '[EMAIL PROTECTED]' % (listname, domain, maildir)
for command in configuration.smtp.command:
print '[EMAIL PROTECTED]' % (listname, command, domain, maildir)
print >> fp, '# STANZA END'
print >> fp
# Now run the postmap command on the virtual file
os.system(configuration.smtp.alias_command + ' ' + fp.name)
--- NEW FILE: verp.py ---
# Copyright (C) 2004 Barry A. Warsaw
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the Free
# Software Foundation; either version 2 of the License, or (at your option)
# any later version.
#
# This program is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
# more details.
#
# You should have received a copy of the GNU General Public License along with
# this program; if not, write to the Free Software Foundation, Inc., 59 Temple
# Place - Suite 330, Boston, MA 02111-1307, USA.
import datetime
import threading
from mailman.smtp.chunkify import sort
from mailman.smtp.decorate import decorate
from mailman.text.safedict import safedict
from mailman.text.strings import mstring
ENVSENDER = '[EMAIL PROTECTED]'
class VERPMailer:
def __init__(self, conn):
self._conn = conn
def send(self, mlist, msg, msgdata):
# There's no point in chunking the recipients for VERP delivery since
# each recipient gets sent in a new SMTP transaction.
#
# See if this is an unshunted message for which there were some
# undelivered recipients.
recips = msgdata.get('undelivered')
if not recips:
recips = msgdata.get('recips')
if not recips:
return
# Prepare the message by personalizing certain parts of the headers,
# and adding the decoration. Then flatten the message, making it
# ready for mail merging in the inner loop.
del msg['to']
msg['To'] = '${mm:recipient}'
decorate(mlist, msg, msgdata)
rawtext = mstring(msg.as_string())
# The substitution dict, with list-specific keys
listname = mlist.listname
d = safedict(
{'mm:public_name': mlist.public_name,
'mm:listname': listname,
'mm:domain': mlist.domain,
'mm:listinfo': 'http://%s/listinfo/%s' % (mydomain, listname),
})
# Get a connection. This will block if there are no connections left.
for recip in recips:
# Calculate member specific keys for the dict
d.update({
'mm:member_page': 'http://%s/options/%s' % (mydomain, recip),
'mm:recipient': recip,
})
# Calculate the VERP'd sender
localpart, domain = recip.split('@', 1)
envsender = ENVSENDER % (localpart, domain, mydomain)
text = rawtext % d
self._conn.send(envsender, [recip], text, msg, msgdata)
_______________________________________________
Mailman-checkins mailing list
[EMAIL PROTECTED]
Unsubscribe: http://mail.python.org/mailman/options/mailman-checkins/archive%40jab.org
