Mark Sapiro pushed to branch master at GNU Mailman / Mailman Core
Commits:
f5b889b6 by Mark Sapiro at 2020-11-13T20:57:51-08:00
Handle/Avoid some UnicodeEncodeErrors when creating digests.
- - - - -
9bc78976 by Mark Sapiro at 2020-11-14T20:02:37+00:00
Merge branch 'digest' into 'master'
Handle/Avoid some UnicodeEncodeErrors when creating digests.
Closes #560
See merge request mailman/mailman!729
- - - - -
5 changed files:
- src/mailman/docs/NEWS.rst
- src/mailman/runners/digest.py
- + src/mailman/runners/tests/data/__init__.py
- + src/mailman/runners/tests/data/ascii_in_utf7.eml
- src/mailman/runners/tests/test_digest.py
Changes:
=====================================
src/mailman/docs/NEWS.rst
=====================================
@@ -8,6 +8,15 @@ Copyright (C) 1998-2018 by the Free Software Foundation, Inc.
Here is a history of user visible changes to Mailman.
+3.3.3
+=====
+
+(202x-xx-xx)
+
+Bugs
+----
+* Handle some UnicodeEncodeErrors in creating digests. (Closes #560)
+
3.3.2
=====
=====================================
src/mailman/runners/digest.py
=====================================
@@ -252,7 +252,9 @@ class RFC1153Digester(Digester):
# multipart message. In that case, just stringify it.
payload = msg.get_payload(decode=True)
if not payload:
- payload = msg.as_string().split('\n\n', 1)[1]
+ # Get the message as bytes to avoid UnicodeEncodeError from
+ # as_string() when charset is incorrectly declared.
+ payload = msg.as_bytes().split(b'\n\n', 1)[1]
if isinstance(payload, bytes):
try:
# Do the decoding inside the try/except so that if the charset
@@ -286,13 +288,14 @@ class RFC1153Digester(Digester):
print(sign_off, file=self._text)
print('*' * len(sign_off), file=self._text)
# If the digest message can't be encoded by the list character set,
- # fall back to utf-8.
+ # fall back to utf-8 with error replacement.
text = self._text.getvalue()
try:
self._message.set_payload(text.encode(self._charset),
charset=self._charset)
except UnicodeError:
- self._message.set_payload(text.encode('utf-8'), charset='utf-8')
+ self._message.set_payload(text.encode('utf-8', errors='replace'),
+ charset='utf-8')
return self._message
=====================================
src/mailman/runners/tests/data/__init__.py
=====================================
=====================================
src/mailman/runners/tests/data/ascii_in_utf7.eml
=====================================
@@ -0,0 +1,34 @@
+From: [email protected]
+To: [email protected]
+Content-Type: text/plain; charset="UTF-7"
+Mime-Version: 1.0
+Content-Transfer-Encoding: 7bit
+
+-----BEGIN PGP SIGNED MESSAGE-----
+Hash: SHA512
+
+On Fri, 2019-03-01 at 08:02 -0800, Mark S wrote:
++AD4 On 3/1/19 5:00 AM, Jim X via Mailman-cabal wrote:
++AD4 +AD4 Can i haz wiki write permissions?
++AD4
++AD4 Yes, and now you do.
+
+Thank you,
+
+- -Jim X.
+-----BEGIN PGP SIGNATURE-----
+
+iQIzBAEBCgAdFiEECPbAhaBWEfiXj/kxdRlcPb+1fkUFAlx5WHkACgkQdRlcPb+1
+fkWe/w/+PEhOx7gyMjHdGT8hr3dZ2Hcr1PQPnM3h1BK5jE2NbWH4OaUotaes7Lo5
+Wjph4Z8UbJJcaFawqdIJgRbt+kK1HVPWysj69F16XfnZ0ANqE6LzSEzax7eNQJoD
+e2LSvs1YtKKRTDtXyjGgr4F2PjK6tEeuIhs7c1b1Nwp27HMFICh2KWAHEoRAuynI
+s98N3lh5llXg+enbj1RqLFkUGZYv+WZ3x/1oD3OwHLt17BGvNBSHhxKayaUq9mwu
+p6y/CI+Gyx8TAD6MiOplSBIq07zTMv5MhJBlqIZLkb5QGP6ClTNLytANh+aVYLqs
+qi4glr6FlcULDe2/GFv1yr4VphNEj7mxvUNBN+oO4ReQA/4asmG7l+aPDv8cpyWA
+yn//Urhh6GXP7zohvkGVzlMmccgJP7gQTgzpCdA3dNEYyQlDcSvjmehjWyUdGLWV
+X70y8PK1jbHyvkH0u9PpdTiG5JhoWFSpcoI0NdGnzL7prXxOl4fflSinxD0Nf7UD
+Y8CViLpmmqxiwFiDYYGXEzkMyCK7F6kZPgL7PVGUrJOvSXVoIBHvVL7J09s6MnvA
+3qZd/LrgaDjjJVwZrdKvwSxPyat4/Jb/pOETfJJyGANFLdLLfbLZfvcI0JPNWluX
+8tViRnap7tR+NpVn/56SoN/6jVUM8Uxtrrtju80oMvwbTYh2Icc=
+=ZGd6
+-----END PGP SIGNATURE-----
=====================================
src/mailman/runners/tests/test_digest.py
=====================================
@@ -21,8 +21,10 @@ import os
import re
import unittest
+from email import message_from_binary_file, message_from_bytes
from email.iterators import _structure as structure
from email.mime.text import MIMEText
+from importlib_resources import open_binary
from io import StringIO
from mailman.app.lifecycle import create_list
from mailman.config import config
@@ -101,6 +103,52 @@ class TestDigest(unittest.TestCase):
self.assertEqual(len(self._shuntq.files), 0, error_log.read())
self._check_virgin_queue()
+ def test_utf7_message_with_inline_ascii_sig(self):
+ # The test message is bigger than 1K. Set the threshold bigger to
+ # avoid double processing in make_digest_messages.
+ self._mlist.digest_size_threshold = 5
+ # Subscribe some users receiving digests.
+ anne = subscribe(self._mlist, 'Anne')
+ anne.preferences.delivery_mode = DeliveryMode.mime_digests
+ bart = subscribe(self._mlist, 'Bart')
+ bart.preferences.delivery_mode = DeliveryMode.plaintext_digests
+ with open_binary('mailman.runners.tests.data',
+ 'ascii_in_utf7.eml') as fp:
+ msg = message_from_binary_file(fp, Message)
+ # Use any error logs as the error message if the test fails.
+ error_log = LogFileMark('mailman.error')
+ make_digest_messages(self._mlist, msg)
+ # The runner will send the file to the shunt queue on exception.
+ self.assertEqual(len(self._shuntq.files), 0, error_log.read())
+ self._check_virgin_queue()
+
+ def test_non_ascii_in_ascii_part(self):
+ # Subscribe some users receiving digests.
+ anne = subscribe(self._mlist, 'Anne')
+ anne.preferences.delivery_mode = DeliveryMode.mime_digests
+ bart = subscribe(self._mlist, 'Bart')
+ bart.preferences.delivery_mode = DeliveryMode.plaintext_digests
+ msg = message_from_bytes(b"""\
+From: [email protected]
+To: [email protected]
+Subject: Non-ascii in ascii message
+MIME-Version: 1.0
+Content-Type: multipart/mixed; boundary="abcxyz"
+
+--abcxyz
+Content-Type: text/plain; charset=us-ascii
+Content-Transfer-Encoding: 8bit
+
+Don\xe2\x80\x99t try this at home.
+--abcxyz--
+""", Message)
+ # Use any error logs as the error message if the test fails.
+ error_log = LogFileMark('mailman.error')
+ make_digest_messages(self._mlist, msg)
+ # The runner will send the file to the shunt queue on exception.
+ self.assertEqual(len(self._shuntq.files), 0, error_log.read())
+ self._check_virgin_queue()
+
def test_mime_digest_format(self):
# Make sure that the format of the MIME digest is as expected.
self._mlist.digest_size_threshold = 0.6
View it on GitLab:
https://gitlab.com/mailman/mailman/-/compare/06f8e366cb008ed41595f1948808352fc7be80ed...9bc78976f5b4d1185cd667552133893f23dd36ce
--
View it on GitLab:
https://gitlab.com/mailman/mailman/-/compare/06f8e366cb008ed41595f1948808352fc7be80ed...9bc78976f5b4d1185cd667552133893f23dd36ce
You're receiving this email because of your account on gitlab.com.
_______________________________________________
Mailman-checkins mailing list -- [email protected]
To unsubscribe send an email to [email protected]
https://mail.python.org/mailman3/lists/mailman-checkins.python.org/
Member address: [email protected]
