Hello community, here is the log from the commit of package offlineimap for openSUSE:Factory checked in at 2016-11-17 12:44:36 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/offlineimap (Old) and /work/SRC/openSUSE:Factory/.offlineimap.new (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "offlineimap" Changes: -------- --- /work/SRC/openSUSE:Factory/offlineimap/offlineimap.changes 2016-10-13 11:32:51.000000000 +0200 +++ /work/SRC/openSUSE:Factory/.offlineimap.new/offlineimap.changes 2016-11-17 12:44:37.000000000 +0100 @@ -1,0 +2,11 @@ +Thu Nov 17 09:13:34 UTC 2016 - [email protected] + +- update to 7.0.9 +* SQLite: make postponing transaction committing possible.. +* UIDMaps: ensure we don't update the map file in dry run mode. +* UIDMaps: prevent from leaving a truncated map file +* Fix flickering in Blinkenlights UI. +* UIDMaps: reorder imports. +* folder: IMAP: remove unused import. + +------------------------------------------------------------------- Old: ---- offlineimap-7.0.8.tar.gz New: ---- offlineimap-7.0.9.tar.gz ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ offlineimap.spec ++++++ --- /var/tmp/diff_new_pack.E87m4r/_old 2016-11-17 12:44:38.000000000 +0100 +++ /var/tmp/diff_new_pack.E87m4r/_new 2016-11-17 12:44:38.000000000 +0100 @@ -19,7 +19,7 @@ %{!?_userunitdir:%define _userunitdir /usr/lib/systemd/user} Name: offlineimap -Version: 7.0.8 +Version: 7.0.9 Release: 0 Summary: Powerful IMAP/Maildir Synchronization Tool License: GPL-2.0+ ++++++ offlineimap-7.0.8.tar.gz -> offlineimap-7.0.9.tar.gz ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/offlineimap-7.0.8/Changelog.md new/offlineimap-7.0.9/Changelog.md --- old/offlineimap-7.0.8/Changelog.md 2016-10-08 10:10:56.000000000 +0200 +++ new/offlineimap-7.0.9/Changelog.md 2016-10-29 10:18:44.000000000 +0200 @@ -15,6 +15,49 @@ * The following excerpt is only usefull when rendered in the website. {:toc} +### OfflineIMAP v7.0.9 (2016-10-29) + +#### Notes + +Let's go for this small but still interesting release. + +The Blinkenlights UI got fixed. Reliability for IMAP/IMAP setups is improved. + +The sqlite backend now honors the fsync configuration option. This allows +commits to the database to be postponed. This might be usefull to disable the +default fsync for some use cases like cache migration from text to sqlite, +syncing after long away periods and more generally when a lot of new email +entries must be written to the cache. + +Because of this change the old fsync option is marked EXPERIMENTAL. However, +setups using the plain text cache are not concerned. Bear in mind that disabling +fsync greatly decreases reliability when resuming from unexpected halts. + +Small code cleanups, too. + +#### Authors + +- Nicolas Sebrecht (4) +- Giel van Schijndel (1) +- Ilias Tsitsimpis (1) + +#### Features + +- SQLite: make postponing transaction committing possible.. [Giel van Schijndel] + +#### Fixes + +- UIDMaps: ensure we don't update the map file in dry run mode. [Nicolas Sebrecht] +- UIDMaps: prevent from leaving a truncated map file. [Nicolas Sebrecht] +- Fix flickering in Blinkenlights UI. [Ilias Tsitsimpis] + +#### Changes + +- UIDMaps: reorder imports. [Nicolas Sebrecht] +- folder: IMAP: remove unused import. [Nicolas Sebrecht] + + + ### OfflineIMAP v7.0.8 (2016-10-08) #### Notes diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/offlineimap-7.0.8/offlineimap/__init__.py new/offlineimap-7.0.9/offlineimap/__init__.py --- old/offlineimap-7.0.8/offlineimap/__init__.py 2016-10-08 10:10:56.000000000 +0200 +++ new/offlineimap-7.0.9/offlineimap/__init__.py 2016-10-29 10:18:44.000000000 +0200 @@ -2,7 +2,7 @@ __productname__ = 'OfflineIMAP' # Expecting trailing "-rcN" or "" for stable releases. -__version__ = "7.0.8" +__version__ = "7.0.9" __copyright__ = "Copyright 2002-2016 John Goerzen & contributors" __author__ = "John Goerzen" __author_email__= "[email protected]" diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/offlineimap-7.0.8/offlineimap/folder/Base.py new/offlineimap-7.0.9/offlineimap/folder/Base.py --- old/offlineimap-7.0.8/offlineimap/folder/Base.py 2016-10-08 10:10:56.000000000 +0200 +++ new/offlineimap-7.0.9/offlineimap/folder/Base.py 2016-10-29 10:18:44.000000000 +0200 @@ -70,6 +70,7 @@ self._sync_deletes = self.config.getdefaultboolean( self.repoconfname, "sync_deletes", True) + self._dofsync = self.config.getdefaultboolean("general", "fsync", True) # Determine if we're running static or dynamic folder filtering # and check filtering status. @@ -103,6 +104,16 @@ # fails if the str is utf-8 return self.name.decode('utf-8') + def __enter__(self): + """Starts a transaction. This will postpone (guaranteed) saving to disk + of all messages saved inside this transaction until its committed.""" + pass + + def __exit__(self, exc_type, exc_val, exc_tb): + """Commits a transaction, all messages saved inside this transaction + will only now be persisted to disk.""" + pass + @property def accountname(self): """Account name as string""" @@ -118,6 +129,9 @@ else: return self.repository.should_sync_folder(self.ffilter_name) + def dofsync(self): + return self._dofsync + def suggeststhreads(self): """Returns True if this folder suggests using threads for actions. @@ -891,38 +905,39 @@ ) return - for num, uid in enumerate(copylist): - # Bail out on CTRL-C or SIGTERM. - if offlineimap.accounts.Account.abort_NOW_signal.is_set(): - break - - if uid == 0: - self.ui.warn("Assertion that UID != 0 failed; ignoring message.") - continue - - if uid > 0 and dstfolder.uidexists(uid): - # dstfolder has message with that UID already, only update status. - flags = self.getmessageflags(uid) - rtime = self.getmessagetime(uid) - statusfolder.savemessage(uid, None, flags, rtime) - continue - - self.ui.copyingmessage(uid, num+1, num_to_copy, self, dstfolder) - # Exceptions are caught in copymessageto(). - if self.suggeststhreads(): - self.waitforthread() - thread = threadutil.InstanceLimitedThread( - self.getinstancelimitnamespace(), - target=self.copymessageto, - name="Copy message from %s:%s"% (self.repository, self), - args=(uid, dstfolder, statusfolder) - ) - thread.start() - threads.append(thread) - else: - self.copymessageto(uid, dstfolder, statusfolder, register=0) - for thread in threads: - thread.join() # Block until all "copy" threads are done. + with self: + for num, uid in enumerate(copylist): + # Bail out on CTRL-C or SIGTERM. + if offlineimap.accounts.Account.abort_NOW_signal.is_set(): + break + + if uid == 0: + self.ui.warn("Assertion that UID != 0 failed; ignoring message.") + continue + + if uid > 0 and dstfolder.uidexists(uid): + # dstfolder has message with that UID already, only update status. + flags = self.getmessageflags(uid) + rtime = self.getmessagetime(uid) + statusfolder.savemessage(uid, None, flags, rtime) + continue + + self.ui.copyingmessage(uid, num+1, num_to_copy, self, dstfolder) + # Exceptions are caught in copymessageto(). + if self.suggeststhreads(): + self.waitforthread() + thread = threadutil.InstanceLimitedThread( + self.getinstancelimitnamespace(), + target=self.copymessageto, + name="Copy message from %s:%s"% (self.repository, self), + args=(uid, dstfolder, statusfolder) + ) + thread.start() + threads.append(thread) + else: + self.copymessageto(uid, dstfolder, statusfolder, register=0) + for thread in threads: + thread.join() # Block until all "copy" threads are done. # Execute new mail hook if we have new mail. if self.have_newmail: diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/offlineimap-7.0.8/offlineimap/folder/IMAP.py new/offlineimap-7.0.9/offlineimap/folder/IMAP.py --- old/offlineimap-7.0.8/offlineimap/folder/IMAP.py 2016-10-08 10:10:56.000000000 +0200 +++ new/offlineimap-7.0.9/offlineimap/folder/IMAP.py 2016-10-29 10:18:44.000000000 +0200 @@ -18,10 +18,9 @@ import random import binascii import re -import os import time -import six from sys import exc_info +import six from .Base import BaseFolder from offlineimap import imaputil, imaplibutil, emailutil, OfflineImapError diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/offlineimap-7.0.8/offlineimap/folder/LocalStatusSQLite.py new/offlineimap-7.0.9/offlineimap/folder/LocalStatusSQLite.py --- old/offlineimap-7.0.8/offlineimap/folder/LocalStatusSQLite.py 2016-10-08 10:10:56.000000000 +0200 +++ new/offlineimap-7.0.9/offlineimap/folder/LocalStatusSQLite.py 2016-10-29 10:18:44.000000000 +0200 @@ -92,6 +92,20 @@ LocalStatusSQLiteFolder.locks[self.filename] = DatabaseFileLock() self._databaseFileLock = LocalStatusSQLiteFolder.locks[self.filename] + self._in_transactions = 0 + + def __enter__(self): + if not self.dofsync(): + assert self.connection is not None + self._in_transactions += 1 + + def __exit__(self, exc_type, exc_val, exc_tb): + if not self.dofsync(): + assert self._in_transactions > 0 + self._in_transactions -= 1 + if self._in_transactions < 1: + self.connection.commit() + def openfiles(self): # Make sure sqlite is in multithreading SERIALIZE mode. assert sqlite.threadsafety == 1, 'Your sqlite is not multithreading safe.' @@ -169,7 +183,8 @@ else: self.connection.execute(sql, args) success = True - self.connection.commit() + if not self._in_transactions: + self.connection.commit() except sqlite.OperationalError as e: if e.args[0] == 'cannot commit - no transaction is active': pass diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/offlineimap-7.0.8/offlineimap/folder/Maildir.py new/offlineimap-7.0.9/offlineimap/folder/Maildir.py --- old/offlineimap-7.0.8/offlineimap/folder/Maildir.py 2016-10-08 10:10:56.000000000 +0200 +++ new/offlineimap-7.0.9/offlineimap/folder/Maildir.py 2016-10-29 10:18:44.000000000 +0200 @@ -60,7 +60,6 @@ def __init__(self, root, name, sep, repository): self.sep = sep # needs to be set before super().__init__ super(MaildirFolder, self).__init__(name, repository) - self.dofsync = self.config.getdefaultboolean("general", "fsync", True) self.root = root # check if we should use a different infosep to support Win file systems self.wincompatible = self.config.getdefaultboolean( @@ -330,7 +329,7 @@ fd.write(content) # Make sure the data hits the disk. fd.flush() - if self.dofsync: + if self.dofsync(): os.fsync(fd) fd.close() diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/offlineimap-7.0.8/offlineimap/folder/UIDMaps.py new/offlineimap-7.0.9/offlineimap/folder/UIDMaps.py --- old/offlineimap-7.0.8/offlineimap/folder/UIDMaps.py 2016-10-08 10:10:56.000000000 +0200 +++ new/offlineimap-7.0.9/offlineimap/folder/UIDMaps.py 2016-10-29 10:18:44.000000000 +0200 @@ -15,10 +15,12 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -import six import os.path +import shutil +from os import fsync, unlink from sys import exc_info from threading import Lock +import six from offlineimap import OfflineImapError from .IMAP import IMAPFolder @@ -31,6 +33,7 @@ be an IMAPFolder. Instance variables (self.): + dryrun: boolean. r2l: dict mapping message uids: self.r2l[remoteuid]=localuid l2r: dict mapping message uids: self.r2l[localuid]=remoteuid #TODO: what is the difference, how are they used? @@ -39,18 +42,33 @@ def __init__(self, *args, **kwargs): IMAPFolder.__init__(self, *args, **kwargs) + self.dryrun = self.config.getdefaultboolean("general", "dry-run", True) self.maplock = Lock() - (self.diskr2l, self.diskl2r) = self._loadmaps() + self.diskr2l, self.diskl2r = self._loadmaps() + self.r2l, self.l2r = None, None + # Representing the local IMAP Folder using local UIDs. + # XXX: This should be removed since we inherit from IMAPFolder. + # See commit 3ce514e92ba7 to know more. self._mb = IMAPFolder(*args, **kwargs) - """Representing the local IMAP Folder using local UIDs""" def _getmapfilename(self): return os.path.join(self.repository.getmapdir(), self.getfolderbasename()) def _loadmaps(self): - with self.maplock: - mapfilename = self._getmapfilename() + mapfilename = self._getmapfilename() + mapfilenametmp = "%s.tmp"% mapfilename + mapfilenamelock = "%s.lock"% mapfilename + with self.maplock and open(mapfilenamelock, 'w') as mapfilelock: + try: + fnctl.lockf(mapfilelock, fnctl.LOCK_EX) # Blocks until acquired. + except NameError: + pass # Windows... + if os.path.exists(mapfilenametmp): + self.ui.warn("a previous run might have leave the UIDMaps file" + " in incorrect state; some sync operations might be done" + " again and some emails might become duplicated.") + unlink(mapfilenametmp) if not os.path.exists(mapfilename): return ({}, {}) file = open(mapfilename, 'rt') @@ -76,9 +94,14 @@ return (r2l, l2r) def _savemaps(self): + if self.dryrun is True: + return + mapfilename = self._getmapfilename() + # Do not use the map file directly to prevent from leaving it truncated. + mapfilenametmp = "%s.tmp"% mapfilename mapfilenamelock = "%s.lock"% mapfilename - with open(mapfilenamelock, 'w') as mapfilelock: + with self.maplock and open(mapfilenamelock, 'w') as mapfilelock: # The "account" lock already prevents from multiple access by # different processes. However, we still need to protect for # multiple access from different threads. @@ -86,10 +109,13 @@ fnctl.lockf(mapfilelock, fnctl.LOCK_EX) # Blocks until acquired. except NameError: pass # Windows... - with open(mapfilename, 'wt') as mapfilefd: + with open(mapfilenametmp, 'wt') as mapfilefd: for (key, value) in self.diskl2r.items(): mapfilefd.write("%d:%d\n"% (key, value)) + if self.dofsync(): + fsync(mapfilefd) # The lock is released when the file descriptor ends. + shutil.move(mapfilenametmp, mapfilename) def _uidlist(self, mapping, items): try: @@ -113,7 +139,6 @@ with self.maplock: # OK. Now we've got a nice list. First, delete things from the # summary that have been deleted from the folder. - for luid in self.diskl2r.keys(): if not luid in reallist: ruid = self.diskl2r[luid] @@ -140,6 +165,7 @@ # Interface from BaseFolder def uidexists(self, ruid): """Checks if the (remote) UID exists in this Folder""" + # This implementation overrides the one in BaseFolder, as it is # much more efficient for the mapped case. return ruid in self.r2l @@ -147,7 +173,9 @@ # Interface from BaseFolder def getmessageuidlist(self): """Gets a list of (remote) UIDs. + You may have to call cachemessagelist() before calling this function!""" + # This implementation overrides the one in BaseFolder, as it is # much more efficient for the mapped case. return self.r2l.keys() @@ -155,16 +183,19 @@ # Interface from BaseFolder def getmessagecount(self): """Gets the number of messages in this folder. + You may have to call cachemessagelist() before calling this function!""" + # This implementation overrides the one in BaseFolder, as it is # much more efficient for the mapped case. return len(self.r2l) # Interface from BaseFolder def getmessagelist(self): - """Gets the current message list. This function's implementation - is quite expensive for the mapped UID case. You must call - cachemessagelist() before calling this function!""" + """Gets the current message list. + + This function's implementation is quite expensive for the mapped UID + case. You must call cachemessagelist() before calling this function!""" retval = {} localhash = self._mb.getmessagelist() @@ -208,6 +239,7 @@ check against dryrun settings, so you need to ensure that savemessage is never called in a dryrun mode. """ + self.ui.savemessage('imap', uid, flags, self) # Mapped UID instances require the source to already have a # positive UID, so simply return here. @@ -263,6 +295,7 @@ :param new_uid: The old remote UID will be changed to a new UID. The UIDMaps case handles this efficiently by simply changing the mappings file.""" + if ruid not in self.r2l: raise OfflineImapError("Cannot change unknown Maildir UID %s"% ruid, OfflineImapError.ERROR.MESSAGE) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/offlineimap-7.0.8/offlineimap/ui/Curses.py new/offlineimap-7.0.9/offlineimap/ui/Curses.py --- old/offlineimap-7.0.8/offlineimap/ui/Curses.py 2016-10-08 10:10:56.000000000 +0200 +++ new/offlineimap-7.0.9/offlineimap/ui/Curses.py 2016-10-29 10:18:44.000000000 +0200 @@ -149,6 +149,7 @@ self.window = curses_win self.acc_num = acc_num self.drawleadstr() + self.ui.exec_locked(self.window.noutrefresh) # Update the child ThreadFrames for child in self.children: child.update(curses_win, self.location, 0) @@ -510,6 +511,7 @@ # received special KEY_RESIZE, resize terminal if key == curses.KEY_RESIZE: self.resizeterm() + return if key < 1 or key > 255: return @@ -574,9 +576,12 @@ self.height, self.width = self.stdscr.getmaxyx() self.logheight = self.height - len(self.accframes) - 1 if resize: - curses.resizeterm(self.height, self.width) + if curses.is_term_resized(self.height, self.width): + curses.resizeterm(self.height, self.width) self.bannerwin.resize(1, self.width) self.logwin.resize(self.logheight, self.width) + self.stdscr.clear() + self.stdscr.noutrefresh() else: self.bannerwin = curses.newwin(1, self.width, 0, 0) self.logwin = curses.newwin(self.logheight, self.width, 1, 0) @@ -621,8 +626,9 @@ else: color = curses.A_NORMAL self.logwin.move(0, 0) - self.logwin.erase() + self.logwin.clear() self.logwin.bkgd(' ', color) + self.logwin.noutrefresh() def getaccountframe(self, acc_name): """Return an AccountFrame() corresponding to acc_name. diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/offlineimap-7.0.8/offlineimap.conf new/offlineimap-7.0.9/offlineimap.conf --- old/offlineimap-7.0.8/offlineimap.conf 2016-10-08 10:10:56.000000000 +0200 +++ new/offlineimap-7.0.9/offlineimap.conf 2016-10-29 10:18:44.000000000 +0200 @@ -164,6 +164,9 @@ # at the expense of greater risk of message duplication in the event of a system # crash or power loss. Default is true. Set it to false to disable fsync. # +# SQLite honors this option since v7.0.8+. However, those SQLite improvements +# are still EXPERIMENTAL. +# #fsync = true
