Update of /cvsroot/mailman/mailman/bin
In directory sc8-pr-cvs1.sourceforge.net:/tmp/cvs-serv18979
Modified Files:
Tag: Release_2_1-maint
update
Log Message:
update_qfiles(), ext_read(), dequeue(): More gross stuff to upgrade the
separate .db and .msg qfiles to the single .pck file.
update_pending(): Updates the global data/pending.pck file to the per-list
.pck files as best as possible.
main(): update the qfiles and pending database.
Index: update
===================================================================
RCS file: /cvsroot/mailman/mailman/bin/update,v
retrieving revision 2.22.2.1
retrieving revision 2.22.2.2
diff -u -d -r2.22.2.1 -r2.22.2.2
--- update 27 Dec 2003 01:00:32 -0000 2.22.2.1
+++ update 10 Feb 2004 22:59:04 -0000 2.22.2.2
@@ -1,6 +1,6 @@
#! @PYTHON@
#
-# Copyright (C) 1998-2003 by the Free Software Foundation, Inc.
+# Copyright (C) 1998-2004 by the Free Software Foundation, Inc.
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
@@ -40,12 +40,17 @@
import errno
import getopt
import shutil
+import cPickle
import marshal
import paths
+import email
+
from Mailman import mm_cfg
from Mailman import Utils
from Mailman import MailList
+from Mailman import Message
+from Mailman import Pending
from Mailman.LockFile import TimeOutError
from Mailman.i18n import _
from Mailman.Queue.Switchboard import Switchboard
@@ -406,17 +411,223 @@
# Be sure the qfiles/in directory exists (we don't really need the
# switchboard object, but it's convenient for creating the directory).
sb = Switchboard(mm_cfg.INQUEUE_DIR)
- for file in os.listdir(mm_cfg.QUEUE_DIR):
- # Updating means just mokving the .db and .msg files to qfiles/in where
+ for filename in os.listdir(mm_cfg.QUEUE_DIR):
+ # Updating means just moving the .db and .msg files to qfiles/in where
# it should be dequeued, converted, and processed normally.
- if file.endswith('.msg'):
- oldmsgfile = os.path.join(mm_cfg.QUEUE_DIR, file)
- newmsgfile = os.path.join(mm_cfg.INQUEUE_DIR, prefix + file)
+ if filename.endswith('.msg'):
+ oldmsgfile = os.path.join(mm_cfg.QUEUE_DIR, filename)
+ newmsgfile = os.path.join(mm_cfg.INQUEUE_DIR, prefix + filename)
os.rename(oldmsgfile, newmsgfile)
- elif file.endswith('.db'):
- olddbfile = os.path.join(mm_cfg.QUEUE_DIR, file)
- newdbfile = os.path.join(mm_cfg.INQUEUE_DIR, prefix + file)
+ elif filename.endswith('.db'):
+ olddbfile = os.path.join(mm_cfg.QUEUE_DIR, filename)
+ newdbfile = os.path.join(mm_cfg.INQUEUE_DIR, prefix + filename)
os.rename(olddbfile, newdbfile)
+ # Now update for the Mailman 2.1.5 qfile format. For every filebase in
+ # the qfiles/* directories that has both a .pck and a .db file, pull the
+ # data out and re-queue them.
+ for dirname in os.listdir(mm_cfg.QUEUE_DIR):
+ dirpath = os.path.join(mm_cfg.QUEUE_DIR, dirname)
+ sb = Switchboard(dirpath)
+ for filename in os.listdir(dirpath):
+ filepath = os.path.join(dirpath, filename)
+ filebase, ext = os.path.splitext(filepath)
+ # Handle the .db metadata files as part of the handling of the
+ # .pck or .msg message files.
+ if ext not in ('.pck', '.msg'):
+ continue
+ msg, data = dequeue(filebase)
+ if msg is not None and data is not None:
+ sb.enqueue(msg, data)
+
+
+
+# Implementations taken from the pre-2.1.5 Switchboard
+def ext_read(filename):
+ fp = open(filename)
+ d = marshal.load(fp)
+ # Update from version 2 files
+ if d.get('version', 0) == 2:
+ del d['filebase']
+ # Do the reverse conversion (repr -> float)
+ for attr in ['received_time']:
+ try:
+ sval = d[attr]
+ except KeyError:
+ pass
+ else:
+ # Do a safe eval by setting up a restricted execution
+ # environment. This may not be strictly necessary since we
+ # know they are floats, but it can't hurt.
+ d[attr] = eval(sval, {'__builtins__': {}})
+ fp.close()
+ return d
+
+
+def dequeue(filebase):
+ # Calculate the .db and .msg filenames from the given filebase.
+ msgfile = os.path.join(filebase + '.msg')
+ pckfile = os.path.join(filebase + '.pck')
+ dbfile = os.path.join(filebase + '.db')
+ # Now we are going to read the message and metadata for the given
+ # filebase. We want to read things in this order: first, the metadata
+ # file to find out whether the message is stored as a pickle or as
+ # plain text. Second, the actual message file. However, we want to
+ # first unlink the message file and then the .db file, because the
+ # qrunner only cues off of the .db file
+ msg = None
+ try:
+ data = ext_read(dbfile)
+ os.unlink(dbfile)
+ except EnvironmentError, e:
+ if e.errno <> errno.ENOENT: raise
+ data = {}
+ # Between 2.1b4 and 2.1b5, the `rejection-notice' key in the metadata
+ # was renamed to `rejection_notice', since dashes in the keys are not
+ # supported in METAFMT_ASCII.
+ if data.has_key('rejection-notice'):
+ data['rejection_notice'] = data['rejection-notice']
+ del data['rejection-notice']
+ msgfp = None
+ try:
+ try:
+ msgfp = open(pckfile)
+ msg = cPickle.load(msgfp)
+ os.unlink(pckfile)
+ except EnvironmentError, e:
+ if e.errno <> errno.ENOENT: raise
+ msgfp = None
+ try:
+ msgfp = open(msgfile)
+ msg = email.message_from_file(msgfp, Message.Message)
+ os.unlink(msgfile)
+ except EnvironmentError, e:
+ if e.errno <> errno.ENOENT: raise
+ except email.Errors.MessageParseError, e:
+ # This message was unparsable, most likely because its
+ # MIME encapsulation was broken. For now, there's not
+ # much we can do about it.
+ print _('message is unparsable: %(filebase)s')
+ msgfp.close()
+ msgfp = None
+ if mm_cfg.QRUNNER_SAVE_BAD_MESSAGES:
+ # Cheapo way to ensure the directory exists w/ the
+ # proper permissions.
+ sb = Switchboard(mm_cfg.BADQUEUE_DIR)
+ os.rename(msgfile, os.path.join(
+ mm_cfg.BADQUEUE_DIR, filebase + '.txt'))
+ else:
+ os.unlink(msgfile)
+ msg = data = None
+ finally:
+ if msgfp:
+ msgfp.close()
+ return msg, data
+
+
+
+def update_pending():
+ file20 = os.path.join(mm_cfg.DATA_DIR, 'pending_subscriptions.db')
+ file214 = os.path.join(mm_cfg.DATA_DIR, 'pending.pck')
+ db = None
+ # Try to load the Mailman 2.0 file
+ try:
+ fp = open(file20)
+ except IOError, e:
+ if e.errno <> errno.ENOENT: raise
+ else:
+ print _('Updating Mailman 2.0 pending_subscriptions.db database')
+ db = marshal.load(fp)
+ # Convert to the pre-Mailman 2.1.5 format
+ db = Pending._update(db)
+ if db is None:
+ # Try to load the Mailman 2.1.x where x < 5, file
+ try:
+ fp = open(file214)
+ except IOError, e:
+ if e.errno <> errno.ENOENT: raise
+ else:
+ print _('Updating Mailman 2.1.4 pending.pck database')
+ db = cPickle.load(fp)
+ # Now upgrade the database to the 2.1.5 format. Each list now has its own
+ # pending.pck file, but only the RE_ENABLE operation actually recorded the
+ # listname in the request. For the SUBSCRIPTION, UNSUBSCRIPTION, and
+ # CHANGE_OF_ADDRESS operations, we know the address of the person making
+ # the request so we can repend this request just for the lists the person
+ # is a member of. For the HELD_MESSAGE operation, we can check the list's
+ # requests.pck file for correlation. Evictions will take care of any
+ # misdirected pendings.
+ reenables_by_list = {}
+ addrops_by_address = {}
+ holds_by_id = {}
+ subs_by_address = {}
+ for key, val in db.items():
+ if key in ('evictions', 'version'):
+ continue
+ try:
+ op = val[0]
+ data = val[1:]
+ except (IndexError, ValueError):
+ print _('Ignoring bad pended data: %(key)s: %(val)s')
+ continue
+ if op in (Pending.UNSUBSCRIPTION, Pending.CHANGE_OF_ADDRESS):
+ # data[0] is the address being unsubscribed
+ addrops_by_address.setdefault(data[0], []).append((key, val))
+ elif op == Pending.SUBSCRIPTION:
+ # data[0] is a UserDesc object
+ addr = data[0].address
+ subs_by_address.setdefault(addr, []).append((key, val))
+ elif op == Pending.RE_ENABLE:
+ # data[0] is the mailing list's internal name
+ reenables_by_list.setdefault(data[0], []).append((key, val))
+ elif op == Pending.HELD_MESSAGE:
+ # data[0] is the hold id. There better only be one entry per id
+ id = data[0]
+ assert not holds_by_id.has_key(id)
+ holds_by_id[id] = (key, val)
+ # Now we have to lock every list and re-pend all the appropriate
+ # requests. Note that this will reset all the expiration dates, but that
+ # should be fine.
+ for listname in Utils.list_names():
+ mlist = MailList.MailList(listname)
+ # This is not the most efficient way to do this because it loads and
+ # saves the pending.pck file each time. :(
+ try:
+ for cookie, data in reenables_by_list.get(listname, []):
+ mlist.pend_repend(cookie, data)
+ for id, (cookie, data) in holds_by_id.items():
+ try:
+ rec = mlist.GetRecord(id)
+ except KeyError:
+ # Not for this list
+ pass
+ else:
+ mlist.pend_repend(cookie, data)
+ del holds_by_id[id]
+ for addr, recs in subs_by_address.items():
+ # We shouldn't have a subscription confirmation if the address
+ # is already a member of the mailing list.
+ if mlist.isMember(addr):
+ continue
+ for cookie, data in recs:
+ mlist.pend_repend(cookie, data)
+ for addr, recs in addrops_by_address.items():
+ # We shouldn't have unsubscriptions or change of address
+ # requests for addresses which aren't members of the list.
+ if not mlist.isMember(addr):
+ continue
+ for cookie, data in recs:
+ mlist.pend_repend(cookie, data)
+ mlist.Save()
+ finally:
+ mlist.Unlock()
+ try:
+ os.unlink(file20)
+ except OSError, e:
+ if e.errno <> errno.ENOENT: raise
+ try:
+ os.unlink(file214)
+ except OSError, e:
+ if e.errno <> errno.ENOENT: raise
@@ -479,31 +690,18 @@
mlist.Unlock()
os.unlink(wmfile)
print _('- usenet watermarks updated and gate_watermarks removed')
- #
- # In Mailman 2.1, the pending database format and file name has changed.
- #
- oldpendingfile = os.path.join(mm_cfg.DATA_DIR, 'pending_subscriptions.db')
- try:
- fp = open(oldpendingfile)
- except IOError, e:
- if e.errno <> errno.ENOENT: raise
- else:
- print _('Updating old pending_subscriptions.db database')
- from Mailman import Pending
- db = marshal.load(fp)
- Pending._update(db)
- fp.close()
- os.unlink(oldpendingfile)
- #
+ # In Mailman 2.1, the pending database format and file name changed, but
+ # in Mailman 2.1.5 it changed again. This should update all existing
+ # files to the 2.1.5 format.
+ update_pending()
# In Mailman 2.1, the qfiles directory has a different structure and a
- # different content.
- #
+ # different content. Also, in Mailman 2.1.5 we collapsed the message
+ # files from separate .msg (pickled Message objects) and .db (marshalled
+ # dictionaries) to a shared .pck file containing two pickles.
update_qfiles()
- #
# This warning was necessary for the upgrade from 1.0b9 to 1.0b10.
# There's no good way of figuring this out for releases prior to 2.0beta2
# :(
- #
if lastversion == NOTFRESH:
print _("""
_______________________________________________
Mailman-checkins mailing list
[EMAIL PROTECTED]
Unsubscribe: http://mail.python.org/mailman/options/mailman-checkins/archive%40jab.org
