Update of /cvsroot/mailman/mailman/Mailman
In directory sc8-pr-cvs1:/tmp/cvs-serv25543/Mailman

Modified Files:
        Defaults.py.in Pending.py 
Log Message:

Robustification in pending.pck handling. In particular, since pending.pck is
shared among all lists, we need to be a lot more careful with locks -- with
thousands of busy (and/or moderated) lists, a lot of lock conflits can
happen. Each pending.pck manipulation is now done inside a loop, and the
loop gets retried in its entirety, starting with grabbing the lockfile and
loading the db, whenever we have a lock failure. Several places now check
more agressively for locks, as the .pck can get very large and take a while
to write, and each process now uses a different temp-filename so as not to
overwrite each other.

This should have no impact on 'average' Mailman installations, with only a
few lists, but should fix (or at least alleviate) Peer's problems with
TimeOutErrors, NotLockedErrors and corrupt pending.pck's, with his Mailman
installation with many, many lists.



Index: Defaults.py.in
===================================================================
RCS file: /cvsroot/mailman/mailman/Mailman/Defaults.py.in,v
retrieving revision 2.115
retrieving revision 2.116
diff -u -d -r2.115 -r2.116
--- Defaults.py.in      13 Mar 2003 05:46:56 -0000      2.115
+++ Defaults.py.in      20 Mar 2003 19:55:29 -0000      2.116
@@ -1065,6 +1065,25 @@
 # the message will be re-queued for later delivery.
 LIST_LOCK_TIMEOUT = seconds(10)
 
+# Set this to true to turn on lock debugging messages for the pending
+# requests database, which will be written to logs/locks.  If you think
+# you're having lock problems, or just want to tune the locks for your
+# system, turn on lock debugging.
+PENDINGDB_LOCK_DEBUGGING = 0
+
+# This variable specifies how long an attempt will be made to acquire a
+# pendingdb lock by the incoming qrunner process.  If the lock acquisition
+# times out, the message will be re-queued for later delivery.
+PENDINGDB_LOCK_TIMEOUT = seconds(30)
+
+# The pendingdb is shared among all lists, and handles all list
+# (un)subscriptions, admin approvals and otherwise held messages, so it is
+# potentially locked a lot more often than single lists. Mailman deals with
+# this by re-trying any attempts to alter the pendingdb that failed because
+# of locking errors. This variable indicates howmany attempt should be made
+# before abandoning all hope.
+PENDINGDB_LOCK_ATTEMPTS = 10
+
 
 
 #####

Index: Pending.py
===================================================================
RCS file: /cvsroot/mailman/mailman/Mailman/Pending.py,v
retrieving revision 2.11
retrieving revision 2.12
diff -u -d -r2.11 -r2.12
--- Pending.py  19 Mar 2003 22:50:49 -0000      2.11
+++ Pending.py  20 Mar 2003 19:55:32 -0000      2.12
@@ -56,29 +56,49 @@
     # It's a programming error if this assertion fails!  We do it this way so
     # the assert test won't fail if the sequence is empty.
     assert content[:1] in _ALLKEYS
-    # Acquire the pending database lock, letting TimeOutError percolate up.
-    lock = LockFile.LockFile(LOCKFILE)
-    lock.lock(timeout=30)
+
+    # Get a lock handle now, but only lock inside the loop.
+    lock = LockFile.LockFile(LOCKFILE,
+                             withlogging=mm_cfg.PENDINGDB_LOCK_DEBUGGING)
+    # We try the main loop several times. If we get a lock error
+    # somewhere (for instance because someone broke the lock) we simply
+    # try again.
+    retries = mm_cfg.PENDINGDB_LOCK_ATTEMPTS
     try:
-        # Load the current database
-        db = _load()
-        # Calculate a unique cookie
-        while 1:
-            n = random.random()
-            now = time.time()
-            hashfood = str(now) + str(n) + str(content)
-            cookie = sha.new(hashfood).hexdigest()
-            if not db.has_key(cookie):
-                break
-        # Store the content, plus the time in the future when this entry will
-        # be evicted from the database, due to staleness.
-        db[cookie] = content
-        evictions = db.setdefault('evictions', {})
-        evictions[cookie] = now + mm_cfg.PENDING_REQUEST_LIFE
-        _save(db)
-        return cookie
+        while retries:
+            retries -= 1
+            if not lock.locked():
+                try:
+                    lock.lock(timeout=mm_cfg.PENDINGDB_LOCK_TIMEOUT)
+                except LockFile.TimeOutError:
+                    continue
+            # Load the current database
+            db = _load()
+            # Calculate a unique cookie
+            while 1:
+                n = random.random()
+                now = time.time()
+                hashfood = str(now) + str(n) + str(content)
+                cookie = sha.new(hashfood).hexdigest()
+                if not db.has_key(cookie):
+                    break
+            # Store the content, plus the time in the future when this
+            # entry will be evicted from the database, due to staleness.
+            db[cookie] = content
+            evictions = db.setdefault('evictions', {})
+            evictions[cookie] = now + mm_cfg.PENDING_REQUEST_LIFE
+            try:
+                _save(db, lock)
+            except LockFile.NotLockedError:
+                continue
+            return cookie
+        else:
+            # We failed to get the lock or keep it long enough to save
+            # the data!
+            raise LockFile.TimeOutError
     finally:
-        lock.unlock(unconditionally=1)
+        if lock.locked():
+            lock.unlock()
 
 
 
@@ -88,30 +108,55 @@
     If optional expunge is true (the default), the record is also removed from
     the database.
     """
-    # Acquire the pending database lock, letting TimeOutError percolate up.
-    # BAW: we perhaps shouldn't acquire the lock if expunge==0.
-    lock = LockFile.LockFile(LOCKFILE)
-    lock.lock(timeout=30)
-    try:
-        # Load the database
+
+    if not expunge:
         db = _load()
         missing = []
         content = db.get(cookie, missing)
         if content is missing:
             return None
-        # Remove the entry from the database
-        if expunge:
+        return content
+
+    # Get a lock handle now, but only lock inside the loop.
+    lock = LockFile.LockFile(LOCKFILE,
+                             withlogging=mm_cfg.PENDINGDB_LOCK_DEBUGGING)
+    # We try the main loop several times. If we get a lock error
+    # somewhere (for instance because someone broke the lock) we simply
+    # try again.
+    retries = mm_cfg.PENDINGDB_LOCK_ATTEMPTS
+    try:
+        while retries:
+            retries -= 1
+            if not lock.locked():
+               try:
+                   lock.lock(timeout=mm_cfg.PENDINGDB_LOCK_TIMEOUT)
+               except LockFile.TimeOutError:
+                   continue
+            # Load the database
+            db = _load()
+            missing = []
+            content = db.get(cookie, missing)
+            if content is missing:
+                return None
             del db[cookie]
             del db['evictions'][cookie]
-            _save(db)
-        return content
+            try:
+                _save(db, lock)
+            except LockFile.NotLockedError:
+                continue
+            return content
+        else:
+            # We failed to get the lock and keep it long enough to save
+            # the data!
+            raise LockFile.TimeOutError
     finally:
-        lock.unlock(unconditionally=1)
+        if lock.locked():
+            lock.unlock()
 
 
 
 def _load():
-    # The list's lock must be acquired.
+    # The list's lock must be acquired if you wish to alter data and save.
     #
     # First try to load the pickle file
     fp = None
@@ -134,8 +179,10 @@
             fp.close()
 
 
-def _save(db):
-    # Lock must be acquired.
+def _save(db, lock):
+    # Lock must be acquired before loading the data that is now being saved.
+    if not lock.locked():
+        raise LockFile.NotLockedError
     evictions = db['evictions']
     now = time.time()
     for cookie, data in db.items():
@@ -154,13 +201,17 @@
     omask = os.umask(007)
     # Always save this as a pickle (safely), and after that succeeds, blow
     # away any old marshal file.
-    tmpfile = PCKFILE + '.tmp'
+    tmpfile = "%s.tmp.%d.%d"%(PCKFILE, os.getpid(), now)
     fp = None
     try:
         fp = open(tmpfile, 'w')
         cPickle.dump(db, fp)
         fp.close()
         fp = None
+        if not lock.locked():
+            # Our lock was broken?
+            os.remove(tmpfile)
+            raise LockFile.NotLockedError
         os.rename(tmpfile, PCKFILE)
         if os.path.exists(DBFILE):
             os.remove(DBFILE)
@@ -173,8 +224,9 @@
 
 def _update(olddb):
     # Update an old pending_subscriptions.db database to the new format
-    lock = LockFile.LockFile(LOCKFILE)
-    lock.lock(timeout=30)
+    lock = LockFile.LockFile(LOCKFILE,
+                             withlogging=mm_cfg.PENDINGDB_LOCK_DEBUGGING)
+    lock.lock(timeout=mm_cfg.PENDINGDB_LOCK_TIMEOUT)
     try:
         # We don't need this entry anymore
         if olddb.has_key('lastculltime'):
@@ -199,6 +251,7 @@
             # request was made.  The new format keeps it as the time the
             # request should be evicted.
             evictions[cookie] = data[-1] + mm_cfg.PENDING_REQUEST_LIFE
-        _save(db)
+        _save(db, lock)
     finally:
-        lock.unlock(unconditionally=1)
+        if lock.locked():
+            lock.unlock()



_______________________________________________
Mailman-checkins mailing list
[EMAIL PROTECTED]
http://mail.python.org/mailman/listinfo/mailman-checkins

Reply via email to