On Thu, Feb 25, 2021 at 11:50:01AM +0000, Sudip Mukherjee wrote: > There is a change merged in upstream which should fix this issue. > Details at: https://github.com/OfflineIMAP/offlineimap3/pull/56 > > It will be great if you can test the latest HEAD from github and > confirm if it fixes the issue. > I can give you a deb package if that is easier.
I have tested this change and it appears to resolve the issue. Attached is the patch that I applied to the current debian/master salsa branch, which should be suitable for incorporating directly into the package as is. I can send it in a salsa MR if you'd like. > But in any case, I think the change is too big to be included in > Debian during this time of the Bullseye freeze. I can add it to > bullseye-backports after the change has been properly tested. Respectfully, this is absolutely not the right response. stable-backports is intended for providing feature updates, and is not the appropriate place to publish fixes for Severity: important bugs. This bug has a significant impact on the ability of this package to fulfill its intended purpose, and further is a regression from buster. Even if bullseye was already released as stable, this bug would warrant a fix in a stable point release. This issue should most definitely be fixed during the bullseye freeze. noah
>From 80b2e21783ae8e3348e8952c3055cf2cd3c500ce Mon Sep 17 00:00:00 2001 From: Noah Meyerhans <[email protected]> Date: Thu, 25 Feb 2021 06:22:03 -0800 Subject: [PATCH] Import upstream fixes for bug 981485 --- debian/patches/bug981485.patch | 977 +++++++++++++++++++++++++++++++++ debian/patches/series | 1 + 2 files changed, 978 insertions(+) create mode 100644 debian/patches/bug981485.patch diff --git a/debian/patches/bug981485.patch b/debian/patches/bug981485.patch new file mode 100644 index 0000000..e44696f --- /dev/null +++ b/debian/patches/bug981485.patch @@ -0,0 +1,977 @@ +Index: offlineimap3/offlineimap/emailutil.py +=================================================================== +--- offlineimap3.orig/offlineimap/emailutil.py ++++ /dev/null +@@ -1,38 +0,0 @@ +-# Some useful functions to extract data out of emails +-# Copyright (C) 2002-2015 John Goerzen & contributors +-# +-# This program is free software; you can redistribute it and/or modify +-# it under the terms of the GNU General Public License as published by +-# the Free Software Foundation; either version 2 of the License, or +-# (at your option) any later version. +-# +-# This program is distributed in the hope that it will be useful, +-# but WITHOUT ANY WARRANTY; without even the implied warranty of +-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +-# GNU General Public License for more details. +-# +-# You should have received a copy of the GNU General Public License +-# along with this program; if not, write to the Free Software +-# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +- +-import email +-from email.parser import Parser as MailParser +- +- +-def get_message_date(content, header='Date'): +- """Parses mail and returns resulting timestamp. +- +- :param content: Mail content +- :param header: the header to extract date from; +- :returns: timestamp or `None` in the case of failure. +- """ +- +- message = MailParser().parsestr(content, True) +- dateheader = message.get(header) +- # parsedate_tz returns a 10-tuple that can be passed to mktime_tz +- # Will be None if missing or not in a valid format. Note that +- # indexes 6, 7, and 8 of the result tuple are not usable. +- datetuple = email.utils.parsedate_tz(dateheader) +- if datetuple is None: +- return None +- return email.utils.mktime_tz(datetuple) +Index: offlineimap3/offlineimap/folder/Base.py +=================================================================== +--- offlineimap3.orig/offlineimap/folder/Base.py ++++ offlineimap3/offlineimap/folder/Base.py +@@ -22,6 +22,11 @@ import re + import time + from sys import exc_info + ++from email import policy ++from email.parser import BytesParser ++from email.generator import BytesGenerator ++from email.utils import parsedate_tz, mktime_tz ++ + from offlineimap import threadutil + from offlineimap.ui import getglobalui + from offlineimap.error import OfflineImapError +@@ -42,6 +47,22 @@ class BaseFolder: + + self.ui = getglobalui() + self.messagelist = {} ++ # Use the built-in email libraries ++ # Establish some policies ++ self.policy = { ++ '7bit': ++ policy.default.clone(cte_type='7bit', utf8=False, refold_source='none'), ++ '7bit-RFC': ++ policy.default.clone(cte_type='7bit', utf8=False, refold_source='none', linesep='\r\n'), ++ '8bit': ++ policy.default.clone(cte_type='8bit', utf8=True, refold_source='none'), ++ '8bit-RFC': ++ policy.default.clone(cte_type='8bit', utf8=True, refold_source='none', linesep='\r\n'), ++ } ++ # Parsers ++ self.parser = {} ++ for key in self.policy: ++ self.parser[key] = BytesParser(policy=self.policy[key]) + # Save original name for folderfilter operations. + self.ffilter_name = name + # Top level dir name is always ''. +@@ -349,7 +370,7 @@ class BaseFolder: + return len(self.getmessagelist()) + + def getmessage(self, uid): +- """Returns the content of the specified message.""" ++ """Returns an email message object.""" + + raise NotImplementedError + +@@ -466,7 +487,7 @@ class BaseFolder: + except: + raise IOError("Can't read %s" % uidfile) + +- def savemessage(self, uid, content, flags, rtime): ++ def savemessage(self, uid, msg, flags, rtime): + """Writes a new message, with the specified uid. + + If the uid is < 0: The backend should assign a new uid and +@@ -637,211 +658,90 @@ class BaseFolder: + for uid in uidlist: + self.deletemessagelabels(uid, labels) + +- def addmessageheader(self, content, linebreak, headername, headervalue): ++ def addmessageheader(self, msg, headername, headervalue): + """Adds new header to the provided message. + +- WARNING: This function is a bit tricky, and modifying it in the wrong +- way, may easily lead to data-loss. +- + Arguments: +- - content: message content, headers and body as a single string +- - linebreak: string that carries line ending ++ - msg: message object + - headername: name of the header to add + - headervalue: value of the header to add + +- .. note:: +- +- The following documentation will not get displayed correctly after +- being processed by Sphinx. View the source of this method to read it. ++ Returns: None + +- This has to deal with strange corner cases where the header is +- missing or empty. Here are illustrations for all the cases, +- showing where the header gets inserted and what the end result +- is. In each illustration, '+' means the added contents. Note +- that these examples assume LF for linebreak, not CRLF, so '\n' +- denotes a linebreak and '\n\n' corresponds to the transition +- between header and body. However if the linebreak parameter +- is set to '\r\n' then you would have to substitute '\r\n' for +- '\n' in the below examples. +- +- * Case 1: No '\n\n', leading '\n' +- +- +X-Flying-Pig-Header: i am here\n +- \n +- This is the body\n +- next line\n +- +- * Case 2: '\n\n' at position 0 +- +- +X-Flying-Pig-Header: i am here +- \n +- \n +- This is the body\n +- next line\n +- +- * Case 3: No '\n\n', no leading '\n' +- +- +X-Flying-Pig-Header: i am here\n +- +\n +- This is the body\n +- next line\n +- +- * Case 4: '\n\n' at non-zero position +- +- Subject: Something wrong with OI\n +- From: [email protected] +- +\nX-Flying-Pig-Header: i am here +- \n +- \n +- This is the body\n +- next line\n + """ + + self.ui.debug('', 'addmessageheader: called to add %s: %s' % + (headername, headervalue)) + +- insertionpoint = content.find(linebreak * 2) +- if insertionpoint == -1: +- self.ui.debug('', 'addmessageheader: headers were missing') +- else: +- self.ui.debug('', +- 'addmessageheader: headers end at position %d' % +- insertionpoint) +- mark = '==>EOH<==' +- contextstart = max(0, insertionpoint - 100) +- contextend = min(len(content), insertionpoint + 100) +- self.ui.debug('', 'addmessageheader: header/body transition " \ +- "context (marked by %s): %s%s%s' % ( +- mark, repr(content[contextstart:insertionpoint]), +- mark, repr(content[insertionpoint:contextend]) +- ) +- ) +- +- # Hoping for case #4. +- prefix = linebreak +- suffix = '' +- # Case #2. +- if insertionpoint == 0: +- prefix = '' +- suffix = '' +- # Either case #1 or #3. +- elif insertionpoint == -1: +- prefix = '' +- suffix = linebreak +- insertionpoint = 0 +- # Case #3: when body starts immediately, without preceding '\n' +- # (this shouldn't happen with proper mail messages, but +- # we seen many broken ones), we should add '\n' to make +- # new (and the only header, in this case) to be properly +- # separated from the message body. +- if content[0:len(linebreak)] != linebreak: +- suffix = suffix + linebreak +- +- self.ui.debug('', +- 'addmessageheader: insertionpoint = %d' % insertionpoint) +- headers = content[0:insertionpoint] +- self.ui.debug('', +- 'addmessageheader: headers = %s' % repr(headers)) +- new_header = prefix + ("%s: %s" % (headername, headervalue)) + suffix +- self.ui.debug('', +- 'addmessageheader: new_header = %s' % repr(new_header)) +- return headers + new_header + content[insertionpoint:] +- +- def __find_eoh(self, content): +- """Searches for the point where mail headers end. +- +- Either double '\n', or end of string. +- +- Arguments: +- - content: contents of the message to search in +- Returns: position of the first non-header byte. +- """ +- +- eoh_cr = content.find('\n\n') +- if eoh_cr == -1: +- eoh_cr = len(content) ++ msg.add_header(headername, headervalue) ++ return + +- return eoh_cr +- +- def getmessageheader(self, content, name): +- """Return the value of the first occurence of the given header. ++ def getmessageheader(self, msg, headername): ++ """Return the value of an undefined occurence of the given header. + + Header name is case-insensitive. + + Arguments: +- - contents: message itself +- - name: name of the header to be searched ++ - msg: message object ++ - headername: name of the header to be searched + + Returns: header value or None if no such header was found. + """ + +- self.ui.debug('', 'getmessageheader: called to get %s' % name) +- eoh = self.__find_eoh(content) +- self.ui.debug('', 'getmessageheader: eoh = %d' % eoh) +- headers = content[0:eoh] +- self.ui.debug('', 'getmessageheader: headers = %s' % repr(headers)) +- +- m = re.search('^%s:(.*)$' % name, headers, +- flags=re.MULTILINE | re.IGNORECASE) +- if m: +- return m.group(1).strip() +- else: +- return None ++ self.ui.debug('', 'getmessageheader: called to get %s' % headername) ++ return msg.get(headername) + +- def getmessageheaderlist(self, content, name): ++ def getmessageheaderlist(self, msg, headername): + """Return a list of values for the given header. + ++ Header name is case-insensitive. ++ + Arguments: +- - contents: message itself +- - name: name of the header to be searched ++ - msg: message object ++ - headername: name of the header to be searched + + Returns: list of header values or empty list if no such header was + found. + """ + +- self.ui.debug('', 'getmessageheaderlist: called to get %s' % name) +- eoh = self.__find_eoh(content) +- self.ui.debug('', 'getmessageheaderlist: eoh = %d' % eoh) +- headers = content[0:eoh] +- self.ui.debug('', 'getmessageheaderlist: headers = %s' % repr(headers)) +- +- return re.findall('^%s:(.*)$' % +- name, headers, flags=re.MULTILINE | re.IGNORECASE) ++ self.ui.debug('', 'getmessageheaderlist: called to get %s' % headername) ++ return msg.get_all(headername, []) + +- def deletemessageheaders(self, content, header_list): +- """Deletes headers in the given list from the message content. ++ def deletemessageheaders(self, msg, header_list): ++ """Deletes headers in the given list from the message. + + Arguments: +- - content: message itself ++ - msg: message object + - header_list: list of headers to be deleted or just the header name + +- We expect our message to have '\n' as line endings.""" ++ """ + + if type(header_list) != type([]): + header_list = [header_list] + self.ui.debug('', + 'deletemessageheaders: called to delete %s' % header_list) + +- if not len(header_list): +- return content ++ for h in header_list: ++ del msg[h] + +- eoh = self.__find_eoh(content) +- self.ui.debug('', 'deletemessageheaders: end of headers = %d' % eoh) +- headers = content[0:eoh] +- rest = content[eoh:] +- self.ui.debug('', 'deletemessageheaders: headers = %s' % repr(headers)) +- new_headers = [] +- for h in headers.split('\n'): +- keep_it = True +- for trim_h in header_list: +- if len(h) > len(trim_h) \ +- and h[0:len(trim_h) + 1] == (trim_h + ":"): +- keep_it = False +- break +- if keep_it: +- new_headers.append(h) ++ return ++ ++ def get_message_date(self, msg, header="Date"): ++ """Returns the Unix timestamp of the email message, derived from the ++ Date field header by default. ++ ++ Arguments: ++ - msg: message object ++ - header: header to extract the date from ++ ++ Returns: timestamp or `None` in the case of failure. ++ """ ++ ++ datetuple = parsedate_tz(msg.get(header)) ++ if datetuple is None: ++ return None + +- return '\n'.join(new_headers) + rest ++ return mktime_tz(datetuple) + + def change_message_uid(self, uid, new_uid): + """Change the message from existing uid to new_uid. +Index: offlineimap3/offlineimap/folder/Gmail.py +=================================================================== +--- offlineimap3.orig/offlineimap/folder/Gmail.py ++++ offlineimap3/offlineimap/folder/Gmail.py +@@ -69,14 +69,13 @@ class GmailFolder(IMAPFolder): + data = self._fetch_from_imap(str(uid), self.retrycount) + + # data looks now e.g. +- # ['320 (X-GM-LABELS (...) UID 17061 BODY[] {2565}','msgbody....'] ++ # ['320 (X-GM-LABELS (...) UID 17061 BODY[] {2565}',<email.message.EmailMessage object>] + # we only asked for one message, and that msg is in data[1]. +- # msbody is in [1]. +- body = data[1].replace("\r\n", "\n") ++ msg = data[1] + + # Embed the labels into the message headers + if self.synclabels: +- m = re.search('X-GM-LABELS\s*[(](.*)[)]', data[0][0]) ++ m = re.search('X-GM-LABELS\s*[(](.*)[)]', data[0]) + if m: + labels = set([imaputil.dequote(lb) for lb in imaputil.imapsplit(m.group(1))]) + else: +@@ -84,19 +83,23 @@ class GmailFolder(IMAPFolder): + labels = labels - self.ignorelabels + labels_str = imaputil.format_labels_string(self.labelsheader, sorted(labels)) + +- # First remove old label headers that may be in the message content retrieved ++ # First remove old label headers that may be in the message body retrieved + # from gmail Then add a labels header with current gmail labels. +- body = self.deletemessageheaders(body, self.labelsheader) +- body = self.addmessageheader(body, '\n', self.labelsheader, labels_str) ++ self.deletemessageheaders(msg, self.labelsheader) ++ self.addmessageheader(msg, self.labelsheader, labels_str) + +- if len(body) > 200: +- dbg_output = "%s...%s" % (str(body)[:150], str(body)[-50:]) +- else: +- dbg_output = body ++ if self.ui.is_debugging('imap'): ++ # Optimization: don't create the debugging objects unless needed ++ msg_s = msg.as_string(policy=self.policy['8bit-RFC']) ++ if len(msg_s) > 200: ++ dbg_output = "%s...%s" % (msg_s[:150], msg_s[-50:]) ++ else: ++ dbg_output = msg_s ++ ++ self.ui.debug('imap', "Returned object from fetching %d: '%s'" % ++ (uid, dbg_output)) + +- self.ui.debug('imap', "Returned object from fetching %d: '%s'" % +- (uid, dbg_output)) +- return body ++ return msg + + def getmessagelabels(self, uid): + if 'labels' in self.messagelist[uid]: +@@ -171,7 +174,7 @@ class GmailFolder(IMAPFolder): + rtime = imaplibutil.Internaldate2epoch(messagestr) + self.messagelist[uid] = {'uid': uid, 'flags': flags, 'labels': labels, 'time': rtime} + +- def savemessage(self, uid, content, flags, rtime): ++ def savemessage(self, uid, msg, flags, rtime): + """Save the message on the Server + + This backend always assigns a new uid, so the uid arg is ignored. +@@ -184,7 +187,7 @@ class GmailFolder(IMAPFolder): + savemessage is never called in a dryrun mode. + + :param uid: Message UID +- :param content: Message content ++ :param msg: Message object + :param flags: Message flags + :param rtime: A timestamp to be used as the mail date + :returns: the UID of the new message as assigned by the server. If the +@@ -193,13 +196,13 @@ class GmailFolder(IMAPFolder): + read-only for example) it will return -1.""" + + if not self.synclabels: +- return super(GmailFolder, self).savemessage(uid, content, flags, rtime) ++ return super(GmailFolder, self).savemessage(uid, msg, flags, rtime) + + labels = set() +- for hstr in self.getmessageheaderlist(content, self.labelsheader): ++ for hstr in self.getmessageheaderlist(msg, self.labelsheader): + labels.update(imaputil.labels_from_header(self.labelsheader, hstr)) + +- ret = super(GmailFolder, self).savemessage(uid, content, flags, rtime) ++ ret = super(GmailFolder, self).savemessage(uid, msg, flags, rtime) + self.savemessagelabels(ret, labels) + return ret + +Index: offlineimap3/offlineimap/folder/GmailMaildir.py +=================================================================== +--- offlineimap3.orig/offlineimap/folder/GmailMaildir.py ++++ offlineimap3/offlineimap/folder/GmailMaildir.py +@@ -90,12 +90,12 @@ class GmailMaildirFolder(MaildirFolder): + if not os.path.exists(filepath): + return set() + +- file = open(filepath, 'rt') +- content = file.read() +- file.close() ++ fd = open(filepath, 'rb') ++ msg = self.parser['8bit'].parse(fd) ++ fd.close() + + self.messagelist[uid]['labels'] = set() +- for hstr in self.getmessageheaderlist(content, self.labelsheader): ++ for hstr in self.getmessageheaderlist(msg, self.labelsheader): + self.messagelist[uid]['labels'].update( + imaputil.labels_from_header(self.labelsheader, hstr)) + self.messagelist[uid]['labels_cached'] = True +@@ -108,7 +108,7 @@ class GmailMaildirFolder(MaildirFolder): + else: + return self.messagelist[uid]['mtime'] + +- def savemessage(self, uid, content, flags, rtime): ++ def savemessage(self, uid, msg, flags, rtime): + """Writes a new message, with the specified uid. + + See folder/Base for detail. Note that savemessage() does not +@@ -116,14 +116,15 @@ class GmailMaildirFolder(MaildirFolder): + savemessage is never called in a dryrun mode.""" + + if not self.synclabels: +- return super(GmailMaildirFolder, self).savemessage(uid, content, ++ return super(GmailMaildirFolder, self).savemessage(uid, msg, + flags, rtime) + + labels = set() +- for hstr in self.getmessageheaderlist(content, self.labelsheader): ++ for hstr in self.getmessageheaderlist(msg, self.labelsheader): + labels.update(imaputil.labels_from_header(self.labelsheader, hstr)) + +- ret = super(GmailMaildirFolder, self).savemessage(uid, content, flags, ++ # TODO - Not sure why the returned uid is stored early as ret here? ++ ret = super(GmailMaildirFolder, self).savemessage(uid, msg, flags, + rtime) + + # Update the mtime and labels. +@@ -145,12 +146,12 @@ class GmailMaildirFolder(MaildirFolder): + filename = self.messagelist[uid]['filename'] + filepath = os.path.join(self.getfullname(), filename) + +- file = open(filepath, 'rt') +- content = file.read() +- file.close() ++ fd = open(filepath, 'rb') ++ msg = self.parser['8bit'].parse(fd) ++ fd.close() + + oldlabels = set() +- for hstr in self.getmessageheaderlist(content, self.labelsheader): ++ for hstr in self.getmessageheaderlist(msg, self.labelsheader): + oldlabels.update(imaputil.labels_from_header(self.labelsheader, + hstr)) + +@@ -167,15 +168,14 @@ class GmailMaildirFolder(MaildirFolder): + sorted(labels | ignoredlabels)) + + # First remove old labels header, and then add the new one. +- content = self.deletemessageheaders(content, self.labelsheader) +- content = self.addmessageheader(content, '\n', self.labelsheader, +- labels_str) ++ self.deletemessageheaders(msg, self.labelsheader) ++ self.addmessageheader(msg, self.labelsheader, labels_str) + + mtime = int(os.stat(filepath).st_mtime) + + # Write file with new labels to a unique file in tmp. + messagename = self.new_message_filename(uid, set()) +- tmpname = self.save_to_tmp_file(messagename, content) ++ tmpname = self.save_to_tmp_file(messagename, msg) + tmppath = os.path.join(self.getfullname(), tmpname) + + # Move to actual location. +Index: offlineimap3/offlineimap/folder/IMAP.py +=================================================================== +--- offlineimap3.orig/offlineimap/folder/IMAP.py ++++ offlineimap3/offlineimap/folder/IMAP.py +@@ -20,7 +20,7 @@ import binascii + import re + import time + from sys import exc_info +-from offlineimap import imaputil, imaplibutil, emailutil, OfflineImapError ++from offlineimap import imaputil, imaplibutil, OfflineImapError + from offlineimap import globals + from imaplib2 import MonthNames + from .Base import BaseFolder +@@ -30,13 +30,6 @@ CRLF = '\r\n' + MSGCOPY_NAMESPACE = 'MSGCOPY_' + + +-# NB: message returned from getmessage() will have '\n' all over the place, +-# NB: there will be no CRLFs. Just before the sending stage of savemessage() +-# NB: '\n' will be transformed back to CRLF. So, for the most parts of the +-# NB: code the stored content will be clean of CRLF and one can rely that +-# NB: line endings will be pure '\n'. +- +- + class IMAPFolder(BaseFolder): + def __init__(self, imapserver, name, repository, decode=True): + # decode the folder name from IMAP4_utf_7 to utf_8 if +@@ -349,19 +342,22 @@ class IMAPFolder(BaseFolder): + data = self._fetch_from_imap(str(uid), self.retrycount) + + # Data looks now e.g. +- # ['320 (17061 BODY[] {2565}','msgbody....'] ++ # ['320 (17061 BODY[] {2565}',<email.message.EmailMessage object>] + # Is a list of two elements. Message is at [1] +- data = data[1].replace(CRLF, "\n") ++ msg = data[1] + +- if len(data) > 200: +- dbg_output = "%s...%s" % (str(data)[:150], str(data)[-50:]) +- else: +- dbg_output = data ++ if self.ui.is_debugging('imap'): ++ # Optimization: don't create the debugging objects unless needed ++ msg_s = msg.as_string(policy=self.policy['8bit-RFC']) ++ if len(msg_s) > 200: ++ dbg_output = "%s...%s" % (msg_s[:150], msg_s[-50:]) ++ else: ++ dbg_output = msg_s + +- self.ui.debug('imap', "Returned object from fetching %d: '%s'" % +- (uid, dbg_output)) ++ self.ui.debug('imap', "Returned object from fetching %d: '%s'" % ++ (uid, dbg_output)) + +- return data ++ return msg + + # Interface from BaseFolder + def getmessagetime(self, uid): +@@ -375,7 +371,7 @@ class IMAPFolder(BaseFolder): + def getmessagekeywords(self, uid): + return self.messagelist[uid]['keywords'] + +- def __generate_randomheader(self, content): ++ def __generate_randomheader(self, msg, policy=None): + """Returns a unique X-OfflineIMAP header + + Generate an 'X-OfflineIMAP' mail header which contains a random +@@ -390,17 +386,21 @@ class IMAPFolder(BaseFolder): + """ + + headername = 'X-OfflineIMAP' ++ if policy is None: ++ output_policy = self.policy['8bit-RFC'] ++ else: ++ output_policy = policy + # We need a random component too. If we ever upload the same + # mail twice (e.g. in different folders), we would still need to + # get the UID for the correct one. As we won't have too many + # mails with identical content, the randomness requirements are + # not extremly critial though. + +- # Compute unsigned crc32 of 'content' as unique hash. ++ # Compute unsigned crc32 of 'msg' (as bytes) into a unique hash. + # NB: crc32 returns unsigned only starting with python 3.0. +- headervalue = str(binascii.crc32(str.encode(content)) +- & 0xffffffff) + '-' +- headervalue += str(self.randomgenerator.randint(0, 9999999999)) ++ headervalue = '{}-{}'.format( ++ (binascii.crc32(msg.as_bytes(policy=output_policy)) & 0xffffffff), ++ self.randomgenerator.randint(0, 9999999999)) + return headername, headervalue + + def __savemessage_searchforheader(self, imapobj, headername, headervalue): +@@ -539,7 +539,7 @@ class IMAPFolder(BaseFolder): + + return 0 + +- def __getmessageinternaldate(self, content, rtime=None): ++ def __getmessageinternaldate(self, msg, rtime=None): + """Parses mail and returns an INTERNALDATE string + + It will use information in the following order, falling back as an +@@ -571,7 +571,7 @@ class IMAPFolder(BaseFolder): + (which is fine as value for append).""" + + if rtime is None: +- rtime = emailutil.get_message_date(content) ++ rtime = self.get_message_date(msg) + if rtime is None: + return None + datetuple = time.localtime(rtime) +@@ -619,7 +619,7 @@ class IMAPFolder(BaseFolder): + return internaldate + + # Interface from BaseFolder +- def savemessage(self, uid, content, flags, rtime): ++ def savemessage(self, uid, msg, flags, rtime): + """Save the message on the Server + + This backend always assigns a new uid, so the uid arg is ignored. +@@ -632,7 +632,7 @@ class IMAPFolder(BaseFolder): + savemessage is never called in a dryrun mode. + + :param uid: Message UID +- :param content: Message content ++ :param msg: Message Object + :param flags: Message flags + :param rtime: A timestamp to be used as the mail date + :returns: the UID of the new message as assigned by the server. If the +@@ -647,16 +647,17 @@ class IMAPFolder(BaseFolder): + self.savemessageflags(uid, flags) + return uid + +- content = self.deletemessageheaders(content, self.filterheaders) ++ # Filter user requested headers before uploading to the IMAP server ++ self.deletemessageheaders(msg, self.filterheaders) + +- # Use proper CRLF all over the message. +- content = re.sub("(?<!\r)\n", CRLF, content) ++ # Should just be able to set the policy, to use CRLF in msg output ++ output_policy = self.policy['8bit-RFC'] + + # Get the date of the message, so we can pass it to the server. +- date = self.__getmessageinternaldate(content, rtime) ++ date = self.__getmessageinternaldate(msg, rtime) + + # Message-ID is handy for debugging messages. +- msg_id = self.getmessageheader(content, "message-id") ++ msg_id = self.getmessageheader(msg, "message-id") + if not msg_id: + msg_id = '[unknown message-id]' + +@@ -676,18 +677,20 @@ class IMAPFolder(BaseFolder): + if not use_uidplus: + # Insert a random unique header that we can fetch later. + (headername, headervalue) = self.__generate_randomheader( +- content) ++ msg) + self.ui.debug('imap', 'savemessage: header is: %s: %s' % + (headername, headervalue)) +- content = self.addmessageheader(content, CRLF, +- headername, headervalue) ++ self.addmessageheader(msg, headername, headervalue) + +- if len(content) > 200: +- dbg_output = "%s...%s" % (content[:150], content[-50:]) +- else: +- dbg_output = content +- self.ui.debug('imap', "savemessage: date: %s, content: '%s'" % +- (date, dbg_output)) ++ if self.ui.is_debugging('imap'): ++ # Optimization: don't create the debugging objects unless needed ++ msg_s = msg.as_string(policy=output_policy) ++ if len(msg_s) > 200: ++ dbg_output = "%s...%s" % (msg_s[:150], msg_s[-50:]) ++ else: ++ dbg_output = msg_s ++ self.ui.debug('imap', "savemessage: date: %s, content: '%s'" % ++ (date, dbg_output)) + + try: + # Select folder for append and make the box READ-WRITE. +@@ -695,7 +698,7 @@ class IMAPFolder(BaseFolder): + except imapobj.readonly: + # readonly exception. Return original uid to notify that + # we did not save the message. (see savemessage in Base.py) +- self.ui.msgtoreadonly(self, uid, content, flags) ++ self.ui.msgtoreadonly(self, uid) + return uid + + # Do the APPEND. +@@ -703,7 +706,7 @@ class IMAPFolder(BaseFolder): + (typ, dat) = imapobj.append( + self.getfullIMAPname(), + imaputil.flagsmaildir2imap(flags), +- date, bytes(content, 'utf-8')) ++ date, msg.as_bytes(policy=output_policy)) + # This should only catch 'NO' responses since append() + # will raise an exception for 'BAD' responses: + if typ != 'OK': +@@ -716,12 +719,12 @@ class IMAPFolder(BaseFolder): + # In this case, we should immediately abort + # the repository sync and continue + # with the next account. +- msg = \ ++ err_msg = \ + "Saving msg (%s) in folder '%s', " \ + "repository '%s' failed (abort). " \ + "Server responded: %s %s\n" % \ + (msg_id, self, self.getrepository(), typ, dat) +- raise OfflineImapError(msg, OfflineImapError.ERROR.REPO) ++ raise OfflineImapError(err_msg, OfflineImapError.ERROR.REPO) + retry_left = 0 # Mark as success. + except imapobj.abort as e: + # Connection has been reset, release connection and retry. +@@ -832,7 +835,7 @@ class IMAPFolder(BaseFolder): + """Fetches data from IMAP server. + + Arguments: +- - uids: message UIDS ++ - uids: message UIDS (OfflineIMAP3: First UID returned only) + - retry_num: number of retries to make + + Returns: data obtained by this query.""" +@@ -888,9 +891,21 @@ class IMAPFolder(BaseFolder): + "with UID '%s'" % (self.getrepository(), uids) + raise OfflineImapError(reason, severity) + +- # Convert bytes to str ++ # JI: In offlineimap, this function returned a tuple of strings for each ++ # fetched UID, offlineimap3 calls to the imap object return bytes and so ++ # originally a fixed, utf-8 conversion was done and *only* the first ++ # response (d[0]) was returned. Note that this alters the behavior ++ # between code bases. However, it seems like a single UID is the intent ++ # of this function so retaining the modfication here for now. ++ # ++ # TODO: Can we assume the server response containing the meta data is ++ # always 'utf-8' encoded? Assuming yes for now. ++ # ++ # Convert responses, d[0][0], into a 'utf-8' string (from bytes) and ++ # Convert email, d[0][1], into a message object (from bytes) ++ + ndata0 = data[0][0].decode('utf-8') +- ndata1 = data[0][1].decode('utf-8', errors='replace') ++ ndata1 = self.parser['8bit-RFC'].parsebytes(data[0][1]) + ndata = [ndata0, ndata1] + + return ndata +Index: offlineimap3/offlineimap/folder/LocalStatus.py +=================================================================== +--- offlineimap3.orig/offlineimap/folder/LocalStatus.py ++++ offlineimap3/offlineimap/folder/LocalStatus.py +@@ -190,7 +190,7 @@ class LocalStatusFolder(BaseFolder): + os.close(fd) + + # Interface from BaseFolder +- def savemessage(self, uid, content, flags, rtime, mtime=0, labels=None): ++ def savemessage(self, uid, msg, flags, rtime, mtime=0, labels=None): + """Writes a new message, with the specified uid. + + See folder/Base for detail. Note that savemessage() does not +Index: offlineimap3/offlineimap/folder/LocalStatusSQLite.py +=================================================================== +--- offlineimap3.orig/offlineimap/folder/LocalStatusSQLite.py ++++ offlineimap3/offlineimap/folder/LocalStatusSQLite.py +@@ -323,7 +323,7 @@ class LocalStatusSQLiteFolder(BaseFolder + # assert False,"getmessageflags() called on non-existing message" + + # Interface from BaseFolder +- def savemessage(self, uid, content, flags, rtime, mtime=0, labels=None): ++ def savemessage(self, uid, msg, flags, rtime, mtime=0, labels=None): + """Writes a new message, with the specified uid. + + See folder/Base for detail. Note that savemessage() does not +Index: offlineimap3/offlineimap/folder/Maildir.py +=================================================================== +--- offlineimap3.orig/offlineimap/folder/Maildir.py ++++ offlineimap3/offlineimap/folder/Maildir.py +@@ -23,7 +23,7 @@ import os + from sys import exc_info + from threading import Lock + from hashlib import md5 +-from offlineimap import OfflineImapError, emailutil ++from offlineimap import OfflineImapError + from .Base import BaseFolder + + # Find the UID in a message filename +@@ -254,16 +254,14 @@ class MaildirFolder(BaseFolder): + + # Interface from BaseFolder + def getmessage(self, uid): +- """Return the content of the message.""" ++ """Returns an email message object.""" + + filename = self.messagelist[uid]['filename'] + filepath = os.path.join(self.getfullname(), filename) +- file = open(filepath, 'rt') +- retval = file.read() +- file.close() +- # TODO: WHY are we replacing \r\n with \n here? And why do we +- # read it as text? +- return retval.replace("\r\n", "\n") ++ fd = open(filepath, 'rb') ++ retval = self.parser['8bit'].parse(fd) ++ fd.close() ++ return retval + + # Interface from BaseFolder + def getmessagetime(self, uid): +@@ -288,17 +286,21 @@ class MaildirFolder(BaseFolder): + uid, self._foldermd5, self.infosep, ''.join(sorted(flags))) + return uniq_name.replace(os.path.sep, self.sep_subst) + +- def save_to_tmp_file(self, filename, content): +- """Saves given content to the named temporary file in the ++ def save_to_tmp_file(self, filename, msg, policy=None): ++ """Saves given message to the named temporary file in the + 'tmp' subdirectory of $CWD. + + Arguments: + - filename: name of the temporary file; +- - content: data to be saved. ++ - msg: Email message object + + Returns: relative path to the temporary file + that was created.""" + ++ if policy is None: ++ output_policy = self.policy['8bit'] ++ else: ++ output_policy = policy + tmpname = os.path.join('tmp', filename) + # Open file and write it out. + # XXX: why do we need to loop 7 times? +@@ -324,8 +326,8 @@ class MaildirFolder(BaseFolder): + else: + raise + +- fd = os.fdopen(fd, 'wt') +- fd.write(content) ++ fd = os.fdopen(fd, 'wb') ++ fd.write(msg.as_bytes(policy=output_policy)) + # Make sure the data hits the disk. + fd.flush() + if self.dofsync(): +@@ -335,7 +337,7 @@ class MaildirFolder(BaseFolder): + return tmpname + + # Interface from BaseFolder +- def savemessage(self, uid, content, flags, rtime): ++ def savemessage(self, uid, msg, flags, rtime): + """Writes a new message, with the specified uid. + + See folder/Base for detail. Note that savemessage() does not +@@ -359,15 +361,15 @@ class MaildirFolder(BaseFolder): + message_timestamp = None + if self._filename_use_mail_timestamp is not False: + try: +- message_timestamp = emailutil.get_message_date(content, 'Date') ++ message_timestamp = self.get_message_date(msg, 'Date') + if message_timestamp is None: + # Give a try with Delivery-date +- message_timestamp = emailutil.get_message_date( +- content, 'Delivery-date') ++ message_timestamp = self.get_message_date( ++ msg, 'Delivery-date') + except Exception as e: + # This should never happen. + from offlineimap.ui import getglobalui +- datestr = emailutil.get_message_date(content) ++ datestr = self.get_message_date(msg) + ui = getglobalui() + ui.warn("UID %d has invalid date %s: %s\n" + "Not using message timestamp as file prefix" % +@@ -375,11 +377,11 @@ class MaildirFolder(BaseFolder): + # No need to check if message_timestamp is None here since it + # would be overridden by _gettimeseq. + messagename = self.new_message_filename(uid, flags, date=message_timestamp) +- tmpname = self.save_to_tmp_file(messagename, content) ++ tmpname = self.save_to_tmp_file(messagename, msg) + + if self._utime_from_header is True: + try: +- date = emailutil.get_message_date(content, 'Date') ++ date = self.get_message_date(msg, 'Date') + if date is not None: + os.utime(os.path.join(self.getfullname(), tmpname), + (date, date)) +@@ -387,7 +389,7 @@ class MaildirFolder(BaseFolder): + # int32. + except Exception as e: + from offlineimap.ui import getglobalui +- datestr = emailutil.get_message_date(content) ++ datestr = self.get_message_date(msg) + ui = getglobalui() + ui.warn("UID %d has invalid date %s: %s\n" + "Not changing file modification time" % (uid, datestr, e)) +Index: offlineimap3/offlineimap/folder/UIDMaps.py +=================================================================== +--- offlineimap3.orig/offlineimap/folder/UIDMaps.py ++++ offlineimap3/offlineimap/folder/UIDMaps.py +@@ -236,11 +236,11 @@ class MappedIMAPFolder(IMAPFolder): + + # Interface from BaseFolder + def getmessage(self, uid): +- """Returns the content of the specified message.""" ++ """Returns the specified message.""" + return self._mb.getmessage(self.r2l[uid]) + + # Interface from BaseFolder +- def savemessage(self, uid, content, flags, rtime): ++ def savemessage(self, uid, msg, flags, rtime): + """Writes a new message, with the specified uid. + + The UIDMaps class will not return a newly assigned uid, as it +@@ -271,7 +271,7 @@ class MappedIMAPFolder(IMAPFolder): + self.savemessageflags(uid, flags) + return uid + +- newluid = self._mb.savemessage(-1, content, flags, rtime) ++ newluid = self._mb.savemessage(-1, msg, flags, rtime) + if newluid < 1: + raise OfflineImapError("server of repository '%s' did not return " + "a valid UID (got '%s') for UID '%s' from '%s'" % ( +Index: offlineimap3/offlineimap/ui/UIBase.py +=================================================================== +--- offlineimap3.orig/offlineimap/ui/UIBase.py ++++ offlineimap3/offlineimap/ui/UIBase.py +@@ -231,6 +231,9 @@ class UIBase: + else: + self.invaliddebug(debugtype) + ++ def is_debugging(self, debugtype): ++ return (debugtype in self.debuglist) ++ + def debugging(self, debugtype): + global debugtypes + self.logger.debug("Now debugging for %s: %s" % (debugtype, +@@ -266,7 +269,7 @@ class UIBase: + (self.getnicename(x), x.getname()) for x in folder_list]) + + # WARNINGS +- def msgtoreadonly(self, destfolder, uid, content, flags): ++ def msgtoreadonly(self, destfolder, uid): + if self.config.has_option('general', 'ignore-readonly') and \ + self.config.getboolean('general', 'ignore-readonly'): + return diff --git a/debian/patches/series b/debian/patches/series index f5f5d50..1e48ef9 100644 --- a/debian/patches/series +++ b/debian/patches/series @@ -3,3 +3,4 @@ Do-not-use-the-Internet-to-fetch-DTD.patch 0001-Revert-Use-system-sslcacertfile-by-default.patch 0001-BUG-Right-format-for-password-from-Curses.patch 0001-BUG-Gmail-FETCH-error-with-synclabels-enabled.patch +bug981485.patch -- 2.30.0
