------------------------------------------------------------
revno: 1086
committer: Mark Sapiro <[EMAIL PROTECTED]>
branch nick: 2.1
timestamp: Fri 2008-06-06 11:13:09 -0700
message:
Merged the cull_bad_shunt branch.
added:
cron/cull_bad_shunt
modified:
Mailman/Defaults.py.in
NEWS
bin/update
configure
configure.in
cron/Makefile.in
cron/crontab.in.in
------------------------------------------------------------
revno: 1073.1.2
committer: Mark Sapiro <[EMAIL PROTECTED]>
branch nick: cull_bad_shunt
timestamp: Sun 2008-04-27 10:52:35 -0700
message:
Changed bin/update to remove .bak qfiles.
modified:
bin/update
------------------------------------------------------------
revno: 1073.1.1
committer: Mark Sapiro <[EMAIL PROTECTED]>
branch nick: cull_bad_shunt
timestamp: Sun 2008-04-27 10:37:39 -0700
message:
Added new cron/cull_bad_shunt and supporting Defaults.py.in settings.
added:
cron/cull_bad_shunt
renamed:
.bzrignore => .bzrignore
ACKNOWLEDGMENTS => ACKNOWLEDGMENTS
BUGS => BUGS
FAQ => FAQ
INSTALL => INSTALL
Mailman => Mailman
Makefile.in => Makefile.in
NEWS => NEWS
README => README
README-I18N.en => README-I18N.en
README.CONTRIB => README.CONTRIB
README.NETSCAPE => README.NETSCAPE
README.USERAGENT => README.USERAGENT
STYLEGUIDE.txt => STYLEGUIDE.txt
TODO => TODO
UPGRADING => UPGRADING
bin => bin
configure => configure
configure.in => configure.in
contrib => contrib
cron => cron
doc => doc
gnu-COPYING-GPL => gnu-COPYING-GPL
install-sh => install-sh
messages => messages
misc => misc
mkinstalldirs => mkinstalldirs
scripts => scripts
src => src
templates => templates
tests => tests
modified:
Mailman/Defaults.py.in
cron/Makefile.in
cron/bumpdigests*
cron/crontab.in.in*
cron/disabled*
cron/nightly_gzip*
configure
configure.in
=== modified file 'Mailman/Defaults.py.in'
--- a/Mailman/Defaults.py.in 2008-04-14 17:45:27 +0000
+++ b/Mailman/Defaults.py.in 2008-04-27 17:37:39 +0000
@@ -725,6 +725,26 @@
# them in qfiles/bad subdirectory.
QRUNNER_SAVE_BAD_MESSAGES = Yes
+# Depending on the above setting, the queue entries with messages which can't
+# be parsed may be saved in qfiles/bad. Certain other exceptions which occur
+# during unpickling of a queue entry also cause the entry to be saved in
+# qfiles/bad. Various exceptions which occur during message processing cause
+# the message to be shunted (saved in qfiles/shunt) where they can be
+# reprocessed with bin/unshunt after the underlying problem is fixed. The
+# cull_bad_shunt cron job normally runs daily to remove and possibly archive
+# stale entries in qfiles/bad and qfiles/shunt. The following settings
+# control this.
+
+# The length of time after which a qfiles/bad or qfiles/shunt file is
+# considered to be stale. Set to zero to disable culling of qfiles/bad and
+# qfiles/shunt entries.
+BAD_SHUNT_STALE_AFTER = days(7)
+
+# The pathname of a directory (searchable and writable by the Mailman cron
+# user) to which the culled qfiles/bad and qfiles/shunt entries will be
+# moved. Set to None to simply delete the culled entries.
+BAD_SHUNT_ARCHIVE_DIRECTORY = None
+
# This flag causes Mailman to fsync() its data files after writing and
# flushing its contents. While this ensures the data is written to disk,
# avoiding data loss, it may be a performance killer. Note that this flag
=== modified file 'NEWS'
--- a/NEWS 2008-06-06 17:49:41 +0000
+++ b/NEWS 2008-06-06 18:13:09 +0000
@@ -6,6 +6,15 @@
2.x.xx (xx-xxx-xxxx)
+ New Features
+
+ - Added a new cron/cull_bad_shunt script to cull and optionally
+ archive old entries from the bad and shunt queues. This is controlled
+ by new Defaults.py/mm_cfg.py settings BAD_SHUNT_STALE_AFTER (default
+ 7 days) and BAD_SHUNT_ARCHIVE_DIRECTORY (default None) which determine
+ how long to keep bad and shunt queue entries and optionally, where to
+ archive removed entries.
+
Bug fixes and other patches
- Changed the preservation of unparseable messages to be conditional on
@@ -23,6 +32,9 @@
- Changed Utils.ValidateEmail to not allow specials (particularly ':')
in unquoted local parts (1956393).
+ - Changed bin/update to remove .bak files erroneously left behind in
+ qfiles/*/ by a 2.1.9 bug.
+
Miscellaneous
- Brad Knowles' mailman daily status report script updated to 0.0.18.
=== modified file 'bin/update'
--- a/bin/update 2008-05-08 03:42:28 +0000
+++ b/bin/update 2008-06-06 18:13:09 +0000
@@ -442,6 +442,10 @@
for filename in os.listdir(dirpath):
filepath = os.path.join(dirpath, filename)
filebase, ext = os.path.splitext(filepath)
+ # A bug in Mailman 2.1.9 left .bak files behind in some
+ # circumstances. It should be safe to remove them.
+ if ext == '.bak':
+ os.remove(filepath)
# Handle the .db metadata files as part of the handling of the
# .pck or .msg message files.
if ext not in ('.pck', '.msg'):
=== modified file 'configure'
--- a/configure 2007-09-21 22:41:19 +0000
+++ b/configure 2008-06-06 18:13:09 +0000
@@ -4250,6 +4250,7 @@
build/contrib/rotatelogs.py:contrib/rotatelogs.py \
build/cron/bumpdigests:cron/bumpdigests \
build/cron/checkdbs:cron/checkdbs \
+build/cron/cull_bad_shunt:cron/cull_bad_shunt \
build/cron/disabled:cron/disabled \
build/cron/gate_news:cron/gate_news \
build/cron/mailpasswds:cron/mailpasswds \
=== modified file 'configure.in'
--- a/configure.in 2007-09-21 22:22:15 +0000
+++ b/configure.in 2008-06-06 18:13:09 +0000
@@ -593,6 +593,7 @@
contrib/rotatelogs.py \
cron/bumpdigests \
cron/checkdbs \
+cron/cull_bad_shunt \
cron/disabled \
cron/gate_news \
cron/mailpasswds \
=== modified file 'cron/Makefile.in'
--- a/cron/Makefile.in 2005-08-27 01:40:17 +0000
+++ b/cron/Makefile.in 2008-04-27 17:37:39 +0000
@@ -1,4 +1,4 @@
-# Copyright (C) 1998-2003 by the Free Software Foundation, Inc.
+# Copyright (C) 1998-2008 by the Free Software Foundation, Inc.
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
@@ -42,7 +42,7 @@
SHELL= /bin/sh
PROGRAMS= checkdbs mailpasswds senddigests gate_news \
- nightly_gzip bumpdigests disabled
+ nightly_gzip bumpdigests disabled cull_bad_shunt
FILES= crontab.in
BUILDDIR= ../build/cron
=== modified file 'cron/crontab.in.in'
--- a/cron/crontab.in.in 2008-04-28 18:10:45 +0000
+++ b/cron/crontab.in.in 2008-06-06 18:13:09 +0000
@@ -22,3 +22,6 @@
# turn this on if the internal archiver is used and
# GZIP_ARCHIVE_TXT_FILES is false in mm_cfg.py
27 3 * * * @PYTHON@ -S @prefix@/cron/nightly_gzip
+#
+# At 4:30AM daily, cull old entries from the 'bad' and 'shunt' queues.
+30 4 * * * @PYTHON@ -S @prefix@/cron/cull_bad_shunt
=== added file 'cron/cull_bad_shunt'
--- a/cron/cull_bad_shunt 1970-01-01 00:00:00 +0000
+++ b/cron/cull_bad_shunt 2008-04-27 17:37:39 +0000
@@ -0,0 +1,123 @@
+#! @PYTHON@
+#
+# Copyright (C) 2008 by the Free Software Foundation, Inc.
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301,
+# USA.
+
+"""Cull bad and shunt queues, recommended once per day.
+
+This script goes through the 'bad' and 'shunt' queue directories and,
+if mm_cfg.BAD_SHUNT_STALE_AFTER is > 0, it removes all files more than
+that many seconds old.
+
+If mm_cfg.BAD_SHUNT_ARCHIVE_DIRECTORY is a writable directory, the old
+files are moved there. Otherwise they are deleted.
+
+Only regular files immediately subordinate to the 'bad' and 'shunt'
+directories are processed. Anything else is skipped.
+
+Usage: %(PROGRAM)s [options]
+
+Options:
+ -h / --help
+ Print this message and exit.
+"""
+
+import os
+import sys
+import stat
+import time
+import errno
+import getopt
+
+import paths
+# mm_cfg must be imported before the other modules, due to the side-effect of
+# it hacking sys.paths to include site-packages. Without this, running this
+# script from cron with python -S will fail.
+from Mailman import mm_cfg
+from Mailman.i18n import _
+
+# Work around known problems with some RedHat cron daemons
+import signal
+signal.signal(signal.SIGCHLD, signal.SIG_DFL)
+
+PROGRAM = sys.argv[0]
+
+
+
+def usage(code, msg=''):
+ if code:
+ fd = sys.stderr
+ else:
+ fd = sys.stdout
+ print >> fd, _(__doc__)
+ if msg:
+ print >> fd, msg
+ sys.exit(code)
+
+
+
+def main():
+ try:
+ opts, args = getopt.getopt(
+ sys.argv[1:], 'h', ['help'])
+ except getopt.error, msg:
+ usage(1, msg)
+
+ if args:
+ usage(1)
+
+ if mm_cfg.BAD_SHUNT_STALE_AFTER <= 0:
+ # Nothing to do
+ return
+ now = time.time()
+ old = now - float(mm_cfg.BAD_SHUNT_STALE_AFTER)
+ os.stat_float_times(True)
+ adir = mm_cfg.BAD_SHUNT_ARCHIVE_DIRECTORY
+ if (adir and os.access(adir, os.W_OK | os.X_OK)
+ and stat.S_ISDIR(os.stat(adir).st_mode)):
+ pass
+ else:
+ if adir:
+ print >>sys.stderr, '%s: archive directory %s not usable.' % (
+ PROGRAM, adir)
+ adir = None
+ for qdir in (mm_cfg.BADQUEUE_DIR, mm_cfg.SHUNTQUEUE_DIR):
+ try:
+ names = os.listdir(qdir)
+ except OSError, e:
+ if e.errno <> errno.ENOENT:
+ # OK if qdir doesn't exist, else
+ raise
+ continue
+ for name in names:
+ pathname = os.path.join(qdir, name)
+ pstat = os.stat(pathname)
+ if not stat.S_ISREG(pstat.st_mode):
+ # Not a regular file
+ continue
+ if pstat.st_mtime > old:
+ # File is new
+ continue
+ if adir:
+ os.rename(pathname, os.path.join(adir, name))
+ else:
+ os.remove(pathname)
+
+
+
+if __name__ == '__main__':
+ main()
--
Stable, maintained release series
https://code.launchpad.net/~mailman-coders/mailman/2.1
You are receiving this branch notification because you are subscribed to it.
_______________________________________________
Mailman-checkins mailing list
[email protected]
Unsubscribe:
http://mail.python.org/mailman/options/mailman-checkins/archive%40jab.org
