Hello community, here is the log from the commit of package mailprocessing for openSUSE:Factory checked in at 2020-02-13 10:12:29 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/mailprocessing (Old) and /work/SRC/openSUSE:Factory/.mailprocessing.new.26092 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "mailprocessing" Thu Feb 13 10:12:29 2020 rev:3 rq:773711 version:1.2.7 Changes: -------- --- /work/SRC/openSUSE:Factory/mailprocessing/mailprocessing.changes 2019-06-28 16:36:59.198874485 +0200 +++ /work/SRC/openSUSE:Factory/.mailprocessing.new.26092/mailprocessing.changes 2020-02-13 10:12:39.956380002 +0100 @@ -1,0 +2,11 @@ +Wed Feb 12 09:49:59 UTC 2020 - Johannes Grassler <[email protected]> + +- Update to mailprocessing-1.2.7 + * imapproc: fix crash when attempting to query message flags on empty folders + * maildirproc: catch and log ENOENT when moving to a nonexistent target folder + * maildirproc: logging improvements/fixes + * maildirproc: fix handling of '/' as separator + * Various test fixes and flake8 cleanups (test suite still non-functional) + * Fix clean target in Makefile on MacOS + +------------------------------------------------------------------- Old: ---- mailprocessing-1.2.6.tar.gz New: ---- mailprocessing-1.2.7.tar.gz ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ mailprocessing.spec ++++++ --- /var/tmp/diff_new_pack.pEv2wF/_old 2020-02-13 10:12:40.468380299 +0100 +++ /var/tmp/diff_new_pack.pEv2wF/_new 2020-02-13 10:12:40.468380299 +0100 @@ -1,7 +1,7 @@ # # spec file for package mailprocessing # -# Copyright (c) 2019 SUSE LINUX GmbH, Nuernberg, Germany. +# Copyright (c) 2020 SUSE LINUX GmbH, Nuernberg, Germany. # # All modifications and additions to the file contributed by third parties # remain the property of their copyright owners, unless otherwise agreed @@ -17,7 +17,7 @@ Name: mailprocessing -Version: 1.2.6 +Version: 1.2.7 Release: 0 Summary: Maildir and IMAP processor/filter using Python 3x as its configuration language License: GPL-2.0-only ++++++ mailprocessing-1.2.6.tar.gz -> mailprocessing-1.2.7.tar.gz ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/mailprocessing-1.2.6/Makefile new/mailprocessing-1.2.7/Makefile --- old/mailprocessing-1.2.6/Makefile 2019-06-27 14:23:38.000000000 +0200 +++ new/mailprocessing-1.2.7/Makefile 2020-02-12 00:11:14.000000000 +0100 @@ -25,6 +25,6 @@ clean: rm -rf maildirproc*-$(VERSION) build dist MANIFEST rm -rf *.gz - find -name '*~' | xargs rm -f + find . -name '*~' | xargs rm -f .PHONY: all clean docs diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/mailprocessing-1.2.6/NEWS new/mailprocessing-1.2.7/NEWS --- old/mailprocessing-1.2.6/NEWS 2019-06-27 14:23:38.000000000 +0200 +++ new/mailprocessing-1.2.7/NEWS 2020-02-12 00:11:14.000000000 +0100 @@ -1,3 +1,12 @@ +Version 1.2.7 (2019-07-20) + + * imapproc: fix crash when attempting to query message flags on empty folders + * maildirproc: catch and log ENOENT when moving to a nonexistent target folder + * maildirproc: logging improvements/fixes + * maildirproc: fix handling of '/' as separator + * Various test fixes and flake8 cleanups (test suite still non-functional) + * Fix clean target in Makefile on MacOS + Version 1.2.6 (2019-06-27) - Added example logrotate configurations diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/mailprocessing-1.2.6/mailprocessing/cmd/imap.py new/mailprocessing-1.2.7/mailprocessing/cmd/imap.py --- old/mailprocessing-1.2.6/mailprocessing/cmd/imap.py 2019-06-27 14:23:38.000000000 +0200 +++ new/mailprocessing-1.2.7/mailprocessing/cmd/imap.py 2020-02-12 00:11:14.000000000 +0100 @@ -35,6 +35,8 @@ from mailprocessing.util import iso_8601_now from mailprocessing.util import write_pidfile +from mailprocessing.version import PKG_VERSION + def main(): imapproc_directory = "~/.mailprocessing" @@ -46,7 +48,7 @@ os.mkdir(os.path.expanduser(imapproc_directory)) parser = OptionParser( - version="1.0.1", + version=PKG_VERSION, description=( "imapproc is a program that scans one or more folders in an" " IMAP mailbox and processes found mail as defined by an rc file." @@ -173,7 +175,7 @@ "--port", type="int", help="IMAP port to use. Defaults to 143 if --use-ssl is not specified " - "and 993 if it is.") + "and 993 if it is.") parser.add_option( "--folder-prefix", type="string", @@ -225,10 +227,10 @@ for opt in ("host", "user"): if not options.__dict__[opt]: - print("Please specify --%s option." % opt, file=sys.stderr) - bad_options = True + print("Please specify --%s option." % opt, file=sys.stderr) + bad_options = True - if not ( options.password or options.password_command ): + if not (options.password or options.password_command): print("Please specify either --password or --password-command.", file=sys.stderr) bad_options = True @@ -258,7 +260,7 @@ sys.stderr.buffer.write(e.output) sys.exit(1) except Exception as e: - print ("Could not execute command %s: %s" % (options.password_command, e)) + print("Could not execute command %s: %s" % (options.password_command, e)) sys.exit(1) processor_kwargs['password'] = p diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/mailprocessing-1.2.6/mailprocessing/cmd/maildir.py new/mailprocessing-1.2.7/mailprocessing/cmd/maildir.py --- old/mailprocessing-1.2.6/mailprocessing/cmd/maildir.py 2019-06-27 14:23:38.000000000 +0200 +++ new/mailprocessing-1.2.7/mailprocessing/cmd/maildir.py 2020-02-12 00:11:14.000000000 +0100 @@ -19,7 +19,7 @@ """maildirproc -- maildir processor -http://mailprocessing.github.io/mailprocessing +http://mailprocessing.github.io/mailprocessing maildirproc is a small program that processes one or several existing mail boxes in the maildir format. It is primarily focused on mail @@ -39,6 +39,9 @@ from mailprocessing.util import iso_8601_now +from mailprocessing.version import PKG_VERSION + + def main(): maildirproc_directory = "~/.mailprocessing" default_rcfile_location = os.path.join(maildirproc_directory, "maildir.rc") @@ -48,7 +51,7 @@ os.mkdir(os.path.expanduser(maildirproc_directory)) parser = OptionParser( - version="1.0.1", + version=PKG_VERSION, description=( "maildirproc is a program that scans a number of maildir mail" " boxes and processes found mail as defined by an rc file. See" diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/mailprocessing-1.2.6/mailprocessing/mail/base.py new/mailprocessing-1.2.7/mailprocessing/mail/base.py --- old/mailprocessing-1.2.6/mailprocessing/mail/base.py 2019-06-27 14:23:38.000000000 +0200 +++ new/mailprocessing-1.2.7/mailprocessing/mail/base.py 2020-02-12 00:11:14.000000000 +0100 @@ -17,11 +17,10 @@ # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA # 02110-1301, USA. -import sys - from mailprocessing.mail.target import MailTarget from mailprocessing.mail.header import MailHeader + class MailBase(object): """ This is the base class representing email messages. If you want to create your @@ -79,8 +78,7 @@ "... Mail is not on mailing list {0}".format(list_name)) return False - - ### Methods to implement ### + # Methods to implement ### def copy(self, folder, create=False): """ @@ -176,6 +174,8 @@ metadata (typically a selection of headers) to the log file. """ + message = ("You need to implement a log_processing() method in your " + "MailBase subclass.") raise NotImplementedError(message) def is_seen(self): diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/mailprocessing-1.2.6/mailprocessing/mail/dryrun.py new/mailprocessing-1.2.7/mailprocessing/mail/dryrun.py --- old/mailprocessing-1.2.6/mailprocessing/mail/dryrun.py 2019-06-27 14:23:38.000000000 +0200 +++ new/mailprocessing-1.2.7/mailprocessing/mail/dryrun.py 2020-02-12 00:11:14.000000000 +0100 @@ -17,16 +17,16 @@ # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA # 02110-1301, USA. -from mailprocessing.mail.base import MailBase from mailprocessing.mail.imap import ImapMail from mailprocessing.mail.maildir import MaildirMail + class DryRunImap(ImapMail): def __init__(self, *args, **kwargs): super(DryRunImap, self).__init__(*args, **kwargs) def copy(self, maildir, **kwargs): - maildir = self.processor.list_path(maildir, + maildir = self.processor.list_path(maildir, sep=self.processor.separator) self._processor.log("==> Copying to {0}".format(maildir)) @@ -40,14 +40,14 @@ self._forward(False, addresses, env_sender) def move(self, maildir, **kwargs): - maildir = self.processor.list_path(maildir, + maildir = self.processor.list_path(maildir, sep=self.processor.separator) self._processor.log("==> Moving to {0}".format(maildir)) # ---------------------------------------------------------------- def _forward(self, delete, addresses, env_sender): - if isinstance(addresses, basestring): + if isinstance(addresses, str): addresses = [addresses] else: addresses = list(addresses) @@ -68,7 +68,7 @@ super(DryRunMaildir, self).__init__(*args, **kwargs) def copy(self, maildir, **kwargs): - maildir = self.processor.list_path(maildir, + maildir = self.processor.list_path(maildir, sep=self.processor.separator) self._processor.log("==> Copying to {0}".format(maildir)) @@ -82,14 +82,14 @@ self._forward(False, addresses, env_sender) def move(self, maildir, **kwargs): - maildir = self.processor.list_path(maildir, + maildir = self.processor.list_path(maildir, sep=self.processor.separator) self._processor.log("==> Moving to {0}".format(maildir)) # ---------------------------------------------------------------- def _forward(self, delete, addresses, env_sender): - if isinstance(addresses, basestring): + if isinstance(addresses, str): addresses = [addresses] else: addresses = list(addresses) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/mailprocessing-1.2.6/mailprocessing/mail/header.py new/mailprocessing-1.2.7/mailprocessing/mail/header.py --- old/mailprocessing-1.2.6/mailprocessing/mail/header.py 2019-06-27 14:23:38.000000000 +0200 +++ new/mailprocessing-1.2.7/mailprocessing/mail/header.py 2020-02-12 00:11:14.000000000 +0100 @@ -18,7 +18,7 @@ # 02110-1301, USA. import re -import sys + class MailHeader(object): def __init__(self, mail, name, text): diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/mailprocessing-1.2.6/mailprocessing/mail/imap.py new/mailprocessing-1.2.7/mailprocessing/mail/imap.py --- old/mailprocessing-1.2.6/mailprocessing/mail/imap.py 2019-06-27 14:23:38.000000000 +0200 +++ new/mailprocessing-1.2.7/mailprocessing/mail/imap.py 2020-02-12 00:11:14.000000000 +0100 @@ -18,9 +18,7 @@ # 02110-1301, USA. import imaplib -import re import subprocess -import sys from email import errors as email_errors from email import header as email_header @@ -29,6 +27,7 @@ from mailprocessing.mail.base import MailBase from mailprocessing import signals + class ImapMail(MailBase): """ This class is used for processing emails in IMAP mailboxes. It is chiefly @@ -92,7 +91,7 @@ status, data = self._processor.imap.uid('copy', self.uid, folder) except self._processor.imap.error as e: self._processor.fatal_imap_error("Copying message UID %s to %s" - % (self.uid, folder), e) + % (self.uid, folder), e) if status == 'NO': if create and 'TRYCREATE' in data[0].decode('ascii'): self._processor.log("==> Destination folder %s does not exist, " @@ -103,7 +102,7 @@ status, data = self._processor.imap.uid('copy', self.uid, folder) except self._processor.imap.error as e: self._processor.fatal_imap_error("Copying message UID %s to %s" - % (self.uid, folder), e) + % (self.uid, folder), e) if status == 'NO': self._processor.fatal_imap_error("Copying message UID %s " " to %s failed with NO, " @@ -140,7 +139,7 @@ # Fail hard because repeated failures here can leave a mess of # messages with `Deleted` flags. self._processor.fatal_imap_error("Deleting message %s'" % - self.uid, e) + self.uid, e) raise # make sure this gets purged from cache later @@ -161,7 +160,7 @@ if self.processor.selected != folder: self._processor.select(self.folder) - if isinstance(addresses, basestring): + if isinstance(addresses, str): addresses = [addresses] else: addresses = list(addresses) @@ -186,11 +185,10 @@ return p = subprocess.Popen( - "{0} {1} -- {2}".format( - self._processor.sendmail, - flags, - " ".join(addresses) - ), + "{0} {1} -- {2}".format(self._processor.sendmail, + flags, + " ".join(addresses) + ), shell=True, stdin=subprocess.PIPE) @@ -295,36 +293,35 @@ flags = imaplib.ParseFlags(data[0][0]) if self.message_flags is None: - self.message_flags = [] - for flag in flags: - self.message_flags.append(flag.decode('ascii')) - + self.message_flags = [] + for flag in flags: + self.message_flags.append(flag.decode('ascii')) headers = email_parser.Parser().parsestr(data[0][1].decode('ascii', 'ignore'), headersonly=True) if self._headers is None: - self._headers = {} - for name in headers.keys(): - value_parts = [] - for header in headers.get_all(name, []): - try: - for (s, c) in email_header.decode_header(header): - # email.header.decode_header in Python 3.x may - # return either [(str, None)] or [(bytes, - # None), ..., (bytes, encoding)]. We must - # compensate for this. - if not isinstance(s, str): - s = s.decode(c if c else "ascii") - value_parts.append(s) - except (email_errors.HeaderParseError, LookupError, - ValueError): - self._processor.log_error( - "Error: Could not decode header {0} in message " - "UID {1}".format(ascii(header), self.uid)) - value_parts.append(header) - self._headers[name.lower()] = " ".join(value_parts) + self._headers = {} + for name in headers.keys(): + value_parts = [] + for header in headers.get_all(name, []): + try: + for (s, c) in email_header.decode_header(header): + # email.header.decode_header in Python 3.x may + # return either [(str, None)] or [(bytes, + # None), ..., (bytes, encoding)]. We must + # compensate for this. + if not isinstance(s, str): + s = s.decode(c if c else "ascii") + value_parts.append(s) + except (email_errors.HeaderParseError, LookupError, + ValueError): + self._processor.log_error( + "Error: Could not decode header {0} in message " + "UID {1}".format(ascii(header), self.uid)) + value_parts.append(header) + self._headers[name.lower()] = " ".join(value_parts) return True diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/mailprocessing-1.2.6/mailprocessing/mail/maildir.py new/mailprocessing-1.2.7/mailprocessing/mail/maildir.py --- old/mailprocessing-1.2.6/mailprocessing/mail/maildir.py 2019-06-27 14:23:38.000000000 +0200 +++ new/mailprocessing-1.2.7/mailprocessing/mail/maildir.py 2020-02-12 00:11:14.000000000 +0100 @@ -20,7 +20,6 @@ import os import shutil import subprocess -import sys from email import errors as email_errors from email import header as email_header @@ -30,6 +29,7 @@ from mailprocessing.util import iso_8601_now from mailprocessing.util import sha1sum + class MaildirMail(MailBase): def __init__(self, processor, **kwargs): self._maildir = kwargs['maildir'] @@ -90,9 +90,10 @@ self._processor.create_maildir_name() + flagpart) try: self._processor.rename(self.path, target) + self._processor.log("==> Moved: \n(src) {0} -->\n(tgt) {1}".format(self.path, target)) except IOError as e: self._processor.log_io_error( - "Could not rename {0} to {1}".format(tmp_target, target), + "Could not rename {0} to {1}".format(self.path, target), e) def parse_mail(self): @@ -198,7 +199,7 @@ e) def _forward(self, delete, addresses, env_sender): - if isinstance(addresses, basestring): + if isinstance(addresses, str): addresses = [addresses] else: addresses = list(addresses) @@ -220,11 +221,10 @@ return p = subprocess.Popen( - "{0} {1} -- {2}".format( - self._processor.sendmail, - flags, - " ".join(addresses) - ), + "{0} {1} -- {2}".format(self._processor.sendmail, + flags, + " ".join(addresses) + ), shell=True, stdin=subprocess.PIPE) shutil.copyfileobj(source_fp, p.stdin) @@ -260,7 +260,6 @@ else: return "" - def _log_processing(self): try: fp = open(self.path, "rb") diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/mailprocessing-1.2.6/mailprocessing/mail/target.py new/mailprocessing-1.2.7/mailprocessing/mail/target.py --- old/mailprocessing-1.2.6/mailprocessing/mail/target.py 2019-06-27 14:23:38.000000000 +0200 +++ new/mailprocessing-1.2.7/mailprocessing/mail/target.py 2020-02-12 00:11:14.000000000 +0100 @@ -17,7 +17,6 @@ # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA # 02110-1301, USA. -import sys class MailTarget(object): _target_headers = ["to", "cc"] diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/mailprocessing-1.2.6/mailprocessing/processor/generic.py new/mailprocessing-1.2.7/mailprocessing/processor/generic.py --- old/mailprocessing-1.2.6/mailprocessing/processor/generic.py 2019-06-27 14:23:38.000000000 +0200 +++ new/mailprocessing-1.2.7/mailprocessing/processor/generic.py 2020-02-12 00:11:14.000000000 +0100 @@ -18,29 +18,23 @@ # 02110-1301, USA. import fcntl -import hashlib -import locale import os -import random -import socket import sys import time from mailprocessing import signals -from mailprocessing.util import iso_8601_now from mailprocessing.util import safe_write + class MailProcessor(object): def __init__( self, rcfile, log_fp, **kwargs): - defaults = { - 'log_level': 1, - 'dry_run': False, - 'run_once': False, - 'auto_reload_rcfile': False - } + defaults = {'log_level': 1, + 'dry_run': False, + 'run_once': False, + 'auto_reload_rcfile': False} for key in defaults: if key not in kwargs: @@ -78,7 +72,7 @@ try: fcntl.flock(self._log_fp, fcntl.LOCK_EX | fcntl.LOCK_NB) lock_acquired = True - except OSError as e: + except OSError: print("Couldn't acquire lock on log file %s, sleeping " "for 5s" % self._log_fp.name, file=sys.stderr) time.sleep(5) @@ -187,13 +181,15 @@ """ Converts path to list form and returns path prepended with the processor's path prefix. If its leading component is already the prefix - or there is no prefix set, the original path in list form is returned. + or there is no prefix set, the original path in list form is returned. """ path = self.path_list(path) if len(path) == 0: return path + if sep == '/': + return path # Special case (mostly relevant for maildirs): # Return an empty first component if path and # prefix are identical. diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/mailprocessing-1.2.6/mailprocessing/processor/imap.py new/mailprocessing-1.2.7/mailprocessing/processor/imap.py --- old/mailprocessing-1.2.6/mailprocessing/processor/imap.py 2019-06-27 14:23:38.000000000 +0200 +++ new/mailprocessing-1.2.7/mailprocessing/processor/imap.py 2020-02-12 00:11:14.000000000 +0100 @@ -17,18 +17,12 @@ # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA # 02110-1301, USA. -import codecs import errno import imaplib import json -import locale -import os -import random -import socket import ssl import re import sys -import time from email import errors as email_errors from email import header as email_header @@ -37,13 +31,12 @@ from mailprocessing import signals from mailprocessing.util import batch_list -from mailprocessing.util import iso_8601_now -from mailprocessing.util import safe_write from mailprocessing.mail.dryrun import DryRunImap from mailprocessing.mail.imap import ImapMail from mailprocessing.processor.generic import MailProcessor + class ImapProcessor(MailProcessor): """ This class is used for processing emails in IMAP mailboxes. It is chiefly @@ -110,7 +103,7 @@ self.cache_delete = {} - if kwargs['folders'] != None: + if kwargs['folders'] is not None: self.set_folders(kwargs['folders']) def authenticate(self): @@ -119,16 +112,14 @@ except self.imap.error as e: self.fatal_imap_error("Login to IMAP server failed", e) - def connect_plain(self): try: self.imap = imaplib.IMAP4(host=self.host, port=self.port) except Exception as e: self.fatal_error("Couldn't connect to IMAP server " - "imap://%s:%d: %s" % ( self.host, self.port, e)) + "imap://%s:%d: %s" % (self.host, self.port, e)) self.authenticate() - def connect_ssl(self, **kwargs): try: self.imap = imaplib.IMAP4_SSL(host=self.host, @@ -136,10 +127,9 @@ ssl_context=self.ssl_context) except Exception as e: self.fatal_error("couldn't connect to imap server " - "imaps://%s:%d: %s" % ( self.host, self.port, e)) + "imaps://%s:%d: %s" % (self.host, self.port, e)) self.authenticate() - def reconnect(self): # Ignore errors when logging out: if the connection is already logged # out, we'll get a bad file descriptor. @@ -169,9 +159,9 @@ self.log("") for folder in self.folders: - if not folder in self.header_cache: + if folder not in self.header_cache: self.header_cache[folder] = {} - if not folder in self.cache_delete: + if folder not in self.cache_delete: self.cache_delete[folder] = [] self._cache_headers() @@ -192,7 +182,6 @@ # ---------------------------------------------------------------- - def create_folder(self, folder, parents=True): """ Creates a new IMAP folder. @@ -264,7 +253,6 @@ return messages - def __iter__(self): """ Iterator method used to invoke the processor from the filter @@ -274,7 +262,6 @@ self.fatal_error("Error: No folders to process") self.rcfile_modified = False - mtime_map = {} while not signals.signal_event.is_set(): if self.auto_reload_rcfile: @@ -304,7 +291,6 @@ self.reconnect() self._cache_headers() - # ---------------------------------------------------------------- def _cache_headers(self): @@ -379,7 +365,7 @@ file. """ - self.log_debug ("Saving header cache to disk.") + self.log_debug("Saving header cache to disk.") # Delete UIDs marked for deletion when the messages where deleted/moved for folder in self.cache_delete: @@ -390,13 +376,13 @@ # Drop unserializable mail objects from header cache before saving. for folder in self.header_cache: if 'uids' in self.header_cache[folder]: - for uid in self.header_cache[folder]['uids']: - try: - self.header_cache[folder]['uids'][uid].pop('obj') - except KeyError as e: - self.log_error("Couldn't drop Mail object for UID %s in " - "folder %s from cache: Mail object " - "missing" % (uid, folder)) + for uid in self.header_cache[folder]['uids']: + try: + self.header_cache[folder]['uids'][uid].pop('obj') + except KeyError: + self.log_error("Couldn't drop Mail object for UID %s in " + "folder %s from cache: Mail object " + "missing" % (uid, folder)) try: f = open(self.cache_file, mode='w') cache = json.dump(cache, f) @@ -405,8 +391,7 @@ self.fatal_error("Couldn't save cache to " "%s: %s" % (self.cache_file, e)) - self.log_debug ("Header cache saved.") - + self.log_debug("Header cache saved.") def _download_headers_batched(self, folder, uids): """ @@ -419,81 +404,78 @@ msgs = {} for batch in batch_list(uids, self.header_batchsize): - uid_list = ",".join(uids) + uid_list = ",".join(uids) + + self.log_debug("==> Downloading headers for UIDs %s" % uid_list) + + try: + self.select(folder) + ret, data = self.imap.uid('fetch', uid_list, + "(FLAGS BODY.PEEK[HEADER])") + except self.imap.error as e: + # Anything imaplib raises an exception for is fatal here. + self.fatal_error("Error retrieving headers for message " + "UIDs %s: %s" % (",".join(uid_list), e)) + + if ret != 'OK': + self.fatal_error( + "Error: Header retrieval failed with status " + "%s for messages: %s" % (ret, ",".join(uid_list))) + + for i in range(0, len(data), 2): + next_decoded = data[i + 1].decode('ascii') + + # The data we are looking for may be in data[i][0] or data[i+1], + # depending on the IMAP server. Let's find out where it is: + + if next_decoded == ')': + # Dovecot + uid_raw = data[i][0].decode('ascii') + + if 'UID' in next_decoded: + # Groupwise + uid_raw = next_decoded - self.log_debug("==> Downloading headers for UIDs %s" % uid_list) + uid = re.search('UID (\d+)', uid_raw).group().split(" ")[1] + + flags = [] + flags_raw = imaplib.ParseFlags(data[i][0]) + + for flag in flags_raw: + flags.append(flag.decode('ascii')) - try: - self.select(folder) - ret, data = self.imap.uid('fetch', uid_list, - "(FLAGS BODY.PEEK[HEADER])") - except self.imap.error as e: - # Anything imaplib raises an exception for is fatal here. - self.fatal_error("Error retrieving headers for message " - "UIDs %s: %s" % (",".join(uid_list), e)) - - if ret != 'OK': - self.fatal_error( - "Error: Header retrieval failed with status " - "%s for messages: %s" % (ret, ",".join(uid_list))) - - - for i in range(0, len(data), 2): - next_decoded = data[i+1].decode('ascii') - - # The data we are looking for may be in data[i][0] or data[i+1], - # depending on the IMAP server. Let's find out where it is: - - if next_decoded == ')': - # Dovecot - uid_raw = data[i][0].decode('ascii') - - if 'UID' in next_decoded: - # Groupwise - uid_raw = next_decoded - - uid = re.search('UID (\d+)', uid_raw).group().split(" ")[1] - - flags = [] - flags_raw = imaplib.ParseFlags(data[i][0]) - - for flag in flags_raw: - flags.append(flag.decode('ascii')) - - - headers_raw = email_parser.Parser().parsestr(data[i][1].decode('ascii', - 'ignore'), - headersonly=True) - headers = {} - for name in headers_raw.keys(): - value_parts = [] - for header in headers_raw.get_all(name, []): - try: - for (s, c) in email_header.decode_header(header): - # email.header.decode_header in Python 3.x may - # return either [(str, None)] or [(bytes, - # None), ..., (bytes, encoding)]. We must - # compensate for this. - if not isinstance(s, str): - s = s.decode(c if c else "ascii") - value_parts.append(s) - except (email_errors.HeaderParseError, LookupError, - ValueError): - self._processor.log_error( - "Error: Could not decode header {0} in message " - "UID {1}".format(ascii(header), uid)) - value_parts.append(header) - headers[name.lower()] = " ".join(value_parts) - - msgs[uid] = {} - msgs[uid]['flags'] = flags - msgs[uid]['headers'] = headers + headers_raw = email_parser.Parser().parsestr(data[i][1].decode('ascii', + 'ignore'), + headersonly=True) + headers = {} + for name in headers_raw.keys(): + value_parts = [] + for header in headers_raw.get_all(name, []): + try: + for (s, c) in email_header.decode_header(header): + # email.header.decode_header in Python 3.x may + # return either [(str, None)] or [(bytes, + # None), ..., (bytes, encoding)]. We must + # compensate for this. + if not isinstance(s, str): + s = s.decode(c if c else "ascii") + value_parts.append(s) + except (email_errors.HeaderParseError, LookupError, + ValueError): + self._processor.log_error( + "Error: Could not decode header {0} in message " + "UID {1}".format(ascii(header), uid)) + value_parts.append(header) + headers[name.lower()] = " ".join(value_parts) + + msgs[uid] = {} + msgs[uid]['flags'] = flags + msgs[uid]['headers'] = headers self.log_debug("==> Header download finished for for UIDs %s" % uid_list) return msgs - def _initialize_cache(self, folder): """ This method initializes a cache data structure for a given folder. @@ -515,7 +497,6 @@ cache[message_uid]['obj'] = msg_obj return cache - def _update_cache(self, folder, cache): """ This method updates an existing header cache data structure for a given @@ -526,10 +507,8 @@ self.log_debug("Updating cache") message_list = self.list_messages(folder) - uids = self.list_messages(folder) uids_download = [] - # Remove stale cache entries stale = [] for message in cache: @@ -545,9 +524,9 @@ cached_headers = cache[message]['headers'] cached_flags = cache[message]['flags'] msg_obj = self._mail_class(self, folder=folder, - uid=message, - headers=cached_headers, - flags=cached_flags) + uid=message, + headers=cached_headers, + flags=cached_flags) cache[message]['obj'] = msg_obj else: @@ -559,16 +538,15 @@ messages = self._download_headers_batched(folder, uids_download) for uid in uids_download: msg_obj = self._mail_class(self, folder=folder, - uid=uid, - headers=messages[uid]['headers'], - flags=messages[uid]['flags']) + uid=uid, + headers=messages[uid]['headers'], + flags=messages[uid]['flags']) cache[uid] = {} cache[uid]['headers'] = msg_obj._headers cache[uid]['flags'] = msg_obj.message_flags cache[uid]['obj'] = msg_obj return cache - def _uidvalidity(self, folder): """ This message returns the IMAP UIDVALIDITY for a given folder. This @@ -596,10 +574,10 @@ self.fatal_error("Couldn't select folder %s: %s" % (folder, e)) if status != 'OK': self.fatal_error("Couldn't select folder %s: %s / %s" % (folder, - status, data[0].decode('ascii'))) + status, data[0].decode('ascii'))) - self.log_debug ("status: %s" % status) - self.log_debug ("data: %s" % data) + self.log_debug("status: %s" % status) + self.log_debug("data: %s" % data) v_string = self.imap.response('UIDVALIDITY')[1][0].decode('ascii') self.uidvalidity[folder] = v_string @@ -643,32 +621,34 @@ else: self.prefix = "" - def get_flags(self, folder): - """ - Get message flags for a folder. - """ - self.log_debug("%d UIDs in cache" % len(self.header_cache[folder]['uids'])) - - uid_list = list(self.header_cache[folder]['uids'].keys()) - for batch in batch_list(uid_list, self.flag_batchsize): - self.log_debug("%d UIDs in batch" % len(batch)) - ret_full = [] - data_full = [] - - try: - self.select(folder) - uids = ",".join(batch) - ret, data = self.imap.uid('fetch', uids, "FLAGS") - ret_full.append(data) - data_full.extend(data) - except self.imap.error as e: - self.fatal_error( - "Could not retrieve message flags for folder {0}, UIDs {1}: {2}" - "{1}".format(folder, uids, e)) + """ + Get message flags for a folder. + """ + self.log_debug("%d UIDs in cache" % len(self.header_cache[folder]['uids'])) + + uid_list = list(self.header_cache[folder]['uids'].keys()) + + if len(uid_list) == 0: + return ([], []) + + for batch in batch_list(uid_list, self.flag_batchsize): + self.log_debug("%d UIDs in batch" % len(batch)) + ret_full = [] + data_full = [] - return (ret_full, data_full) + try: + self.select(folder) + uids = ",".join(batch) + ret, data = self.imap.uid('fetch', uids, "FLAGS") + ret_full.append(data) + data_full.extend(data) + except self.imap.error as e: + self.fatal_error( + "Could not retrieve message flags for folder {0}, UIDs {1}: {2}" + "{1}".format(folder, uids, e)) + return (ret_full, data_full) def refresh_flags(self): """ @@ -690,10 +670,10 @@ flags = [] flags_raw = imaplib.ParseFlags(msg) for flag in flags_raw: - flags.append(flag.decode('ascii')) - uid = m = re.search('UID (\d+)', - msg.decode('ascii')).group().split(" ")[1] - self.log_debug ("UID: %s" % uid) + flags.append(flag.decode('ascii')) + uid = re.search('UID (\d+)', + msg.decode('ascii')).group().split(" ")[1] + self.log_debug("UID: %s" % uid) self.log_debug(" server flags: %s" % flags) self.log_debug(" cache flags: %s" % self.header_cache[folder]['uids'][uid]['flags']) @@ -701,7 +681,6 @@ self.header_cache[folder]['uids'][uid]['obj'].message_flags = flags self.log_info("==> Message flags updated.") - def _status(self, folder): """ Performs an IMAP STATUS on folder. Primarily useful for checking folder @@ -717,7 +696,6 @@ self.select(self.selected) return (status == 'OK', data) - def clean_sleep(self): """ Close IMAP connection and save cache before going to sleep. @@ -730,7 +708,6 @@ self.imap.logout() self.log("==> ...done.") - def clean_exit(self): """ Close connnection and exit in a clean manner. diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/mailprocessing-1.2.6/mailprocessing/processor/maildir.py new/mailprocessing-1.2.7/mailprocessing/processor/maildir.py --- old/mailprocessing-1.2.6/mailprocessing/processor/maildir.py 2019-06-27 14:23:38.000000000 +0200 +++ new/mailprocessing-1.2.7/mailprocessing/processor/maildir.py 2020-02-12 00:11:14.000000000 +0100 @@ -18,22 +18,16 @@ # 02110-1301, USA. import errno -import hashlib -import locale import os import random import socket -import sys import time - -from mailprocessing.util import iso_8601_now -from mailprocessing.util import safe_write - from mailprocessing.processor.generic import MailProcessor from mailprocessing.mail.dryrun import DryRunMaildir from mailprocessing.mail.maildir import MaildirMail + class MaildirProcessor(MailProcessor): def __init__(self, *args, **kwargs): self._maildir_base = None @@ -91,7 +85,7 @@ for mail_file in os.listdir(subdir_path): mail_path = os.path.join(subdir_path, mail_file) yield self._mail_class(self, maildir=maildir, - mail_path=mail_path) + mail_path=mail_path) if self._run_once: break time.sleep(1) @@ -132,21 +126,20 @@ self.log("==> Successfully created folder %s" % target) def create_maildir(self, name, parents=True): - """ - Creates a new maildir. - """ - for d in ['cur', 'new', 'tmp']: - try: - if parents: - os.makedirs(os.path.join(name, d), mode=0o700) - else: - os.mkdir(os.path.join(name, d), mode=0o700) - except OSError as e: - if e.errno == errno.EEXIST: - pass - else: - raise - + """ + Creates a new maildir. + """ + for d in ['cur', 'new', 'tmp']: + try: + if parents: + os.makedirs(os.path.join(name, d), mode=0o700) + else: + os.mkdir(os.path.join(name, d), mode=0o700) + except OSError as e: + if e.errno == errno.EEXIST: + pass + else: + raise def create_maildir_name(self): """Create and return a unique name for a Maildir message.""" @@ -174,4 +167,8 @@ errmsg) def rename(self, source, target): - os.rename(source, target) + try: + os.rename(source, target) + except FileNotFoundError: + self.log_error("Error: Moving file from {0} to {1}. Maybe it doesn't exist?".format( + source, target)) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/mailprocessing-1.2.6/mailprocessing/signals.py new/mailprocessing-1.2.7/mailprocessing/signals.py --- old/mailprocessing-1.2.6/mailprocessing/signals.py 2019-06-27 14:23:38.000000000 +0200 +++ new/mailprocessing-1.2.7/mailprocessing/signals.py 2020-02-12 00:11:14.000000000 +0100 @@ -23,6 +23,7 @@ signal_event = threading.Event() signal_received = None + def handler(signum, frame): global signal_event # used for interruptable sleep if not signum == signal.SIGHUP: @@ -34,9 +35,11 @@ def hup_received(): return signal_received == signal.SIGHUP + def terminate(): return (signal_received == signal.SIGINT) or (signal_received == signal.SIGTERM) + signal.signal(signal.SIGINT, handler) signal.signal(signal.SIGTERM, handler) signal.signal(signal.SIGHUP, handler) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/mailprocessing-1.2.6/mailprocessing/util.py new/mailprocessing-1.2.7/mailprocessing/util.py --- old/mailprocessing-1.2.6/mailprocessing/util.py 2019-06-27 14:23:38.000000000 +0200 +++ new/mailprocessing-1.2.7/mailprocessing/util.py 2020-02-12 00:11:14.000000000 +0100 @@ -25,6 +25,7 @@ basestring = str + def offset_to_timezone(offset): if offset <= 0: sign = "+" @@ -46,19 +47,19 @@ def batch_list(to_batch, batchsize=1000): - """ - Divide large lists. Returns a list of lists with batchsize or fewer items. - """ - - cur = 0 - batches = [] - - while cur <= len(to_batch): - increment = min(len(to_batch) - (cur - 1), batchsize) - batches.append(to_batch[cur:cur + increment]) - cur += increment + """ + Divide large lists. Returns a list of lists with batchsize or fewer items. + """ + + cur = 0 + batches = [] + + while cur <= len(to_batch): + increment = min(len(to_batch) - (cur - 1), batchsize) + batches.append(to_batch[cur:cur + increment]) + cur += increment - return batches + return batches def sha1sum(fp): diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/mailprocessing-1.2.6/mailprocessing/version.py new/mailprocessing-1.2.7/mailprocessing/version.py --- old/mailprocessing-1.2.6/mailprocessing/version.py 1970-01-01 01:00:00.000000000 +0100 +++ new/mailprocessing-1.2.7/mailprocessing/version.py 2020-02-12 00:11:14.000000000 +0100 @@ -0,0 +1,23 @@ +# -*- coding: utf-8; mode: python -*- + +# Copyright (C) 2019 Johannes Grassler <[email protected]> +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA +# 02110-1301, USA. + +# Change the version below to change the mailprocessing/imapproc version. This +# controls the python module's version as well. + +PKG_VERSION = "1.2.7" diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/mailprocessing-1.2.6/run-testsuite new/mailprocessing-1.2.7/run-testsuite --- old/mailprocessing-1.2.6/run-testsuite 2019-06-27 14:23:38.000000000 +0200 +++ new/mailprocessing-1.2.7/run-testsuite 2020-02-12 00:11:14.000000000 +0100 @@ -13,25 +13,21 @@ class TestFixture(unittest.TestCase): _maildir_dir = "test.maildir.%d" % os.getpid() _testdata_dir = "test.testdata.%d" % os.getpid() - _testmail_data_a = [ - "From: Sender <[email protected]>\n", - "To: Recipient <[email protected]>\n", - "Cc: Carbon Copy <[email protected]>\n", - "Subject: A subject with =?utf8?b?csOkdnNtw7ZyZ8Olcw==?=\n", - "Delivered-To: Alpha\n", - "X-BeenThere: Bravo\n", - "X-Mailing-List: Charlie\n", - "Mailing-List: Delta\n", - "\n", - "Body. R\xc3\xa4vsm\xc3\xb6rg\xc3\xa5s.\n", - ] - _testmail_data_bad_header = [ - "From: Sender <[email protected]>\n", - "To: Recipient <[email protected]>\n", - "Subject: =?ISO-2022-JP?B?gYqBiQ=?=\n", - "\n", - "Body.\n", - ] + _testmail_data_a = ["From: Sender <[email protected]>\n", + "To: Recipient <[email protected]>\n", + "Cc: Carbon Copy <[email protected]>\n", + "Subject: A subject with =?utf8?b?csOkdnNtw7ZyZ8Olcw==?=\n", + "Delivered-To: Alpha\n", + "X-BeenThere: Bravo\n", + "X-Mailing-List: Charlie\n", + "Mailing-List: Delta\n", + "\n", + "Body. R\xc3\xa4vsm\xc3\xb6rg\xc3\xa5s.\n"] + _testmail_data_bad_header = ["From: Sender <[email protected]>\n", + "To: Recipient <[email protected]>\n", + "Subject: =?ISO-2022-JP?B?gYqBiQ=?=\n", + "\n", + "Body.\n"] def setUp(self): shutil.rmtree(self._testdata_dir, True) @@ -60,25 +56,20 @@ extra_argv = ["-b", self._maildir_dir] else: extra_argv = ["-b", maildir_base] - argv = [ - "./%s" % self._maildirproc_name, - "--once", - "-l", "/dev/null", - "-r", "-", - ] + extra_argv + argv = ["./%s" % self._maildirproc_name, + "--once", + "-l", "/dev/null", + "-r", "-"] + extra_argv for maildir in maildirs: argv.extend(["-m", maildir]) sh_cmd = "-c 'a=%s/delivered; echo $1 >$a; cat >>$a'" % ( self._testdata_dir) - p = subprocess.Popen( - argv, - stdin=subprocess.PIPE, - env={ - "SENDMAIL": "/bin/sh", - "SENDMAILFLAGS": sh_cmd, - }, - ) - p.communicate(textwrap.dedent(rc)) + p = subprocess.Popen(argv, + stdin=subprocess.PIPE, + env={"SENDMAIL": "/bin/sh", + "SENDMAILFLAGS": sh_cmd}) + dedented_rc = textwrap.dedent(rc) + p.communicate(bytes(dedented_rc, encoding="utf8")) p.wait() assert(p.returncode == 0) @@ -400,13 +391,11 @@ "x-beenthere", "x-mailing-list", "mailing-list"]) self.create("a", "incoming/new/a") self.run_mdp(rc, ["incoming"]) - self.verify_result({ - "incoming/new": 1, - "delivered-to/new": 1, - "x-beenthere/new": 1, - "x-mailing-list/new": 1, - "mailing-list/new": 1, - }) + self.verify_result({"incoming/new": 1, + "delivered-to/new": 1, + "x-beenthere/new": 1, + "x-mailing-list/new": 1, + "mailing-list/new": 1}) class Python2TestSuite(TestSuite): @@ -420,6 +409,7 @@ ###################################################################### + test_runner = unittest.TextTestRunner() test_loader = unittest.defaultTestLoader test_suite = unittest.TestSuite() diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/mailprocessing-1.2.6/setup.py new/mailprocessing-1.2.7/setup.py --- old/mailprocessing-1.2.6/setup.py 2019-06-27 14:23:38.000000000 +0200 +++ new/mailprocessing-1.2.7/setup.py 2020-02-12 00:11:14.000000000 +0100 @@ -2,10 +2,11 @@ # -*-python-*- from setuptools import setup, find_packages +from mailprocessing.version import PKG_VERSION setup( name="mailprocessing", - version="1.2.6", + version=PKG_VERSION, author="Joel Rosdahl", author_email="[email protected]", maintainer="Johannes Grassler",
