Mark Sapiro pushed to branch master at GNU Mailman / Mailman Core
Commits:
94fbd7dd by Mark Sapiro at 2020-02-06T11:36:54-08:00
Fixed html_to_plaintext to properly encode the new payload.
Added test for same and updated existing tests for changes.
- - - - -
3af7ed0b by Mark Sapiro at 2020-02-09T17:08:14+00:00
Merge branch 'fix_677' into 'master'
Fixed html_to_plaintext to properly encode the new payload.
Closes #677
See merge request mailman/mailman!600
- - - - -
4 changed files:
- src/mailman/docs/NEWS.rst
- src/mailman/handlers/docs/filtering.rst
- src/mailman/handlers/mime_delete.py
- src/mailman/handlers/tests/test_mimedel.py
Changes:
=====================================
src/mailman/docs/NEWS.rst
=====================================
@@ -11,7 +11,7 @@ Here is a history of user visible changes to Mailman.
3.3.1
=====
-(2019-XX-XX)
+(2020-XX-XX)
Bugs
----
@@ -41,6 +41,8 @@ Bugs
(Closes #665)
* Residial bounces are now marked as processed. (Closes #669)
* Find members API now searches for the Member's display name too. (Closes
#667)
+* Content filtering ``convert_html_to_plaintext`` no longer creates messages
+ that can't be flattened as_bytes. (Closes #677)
REST
----
=====================================
src/mailman/handlers/docs/filtering.rst
=====================================
@@ -234,8 +234,9 @@ name of the file containing the message payload to filter.
... process(mlist, msg, {})
>>> print(msg.as_string())
From: [email protected]
+ Content-Transfer-Encoding: 7bit
MIME-Version: 1.0
- Content-Type: text/plain
+ Content-Type: text/plain; charset="us-ascii"
X-Content-Filtered-By: Mailman/MimeDel ...
<BLANKLINE>
Converted text/html to text/plain
@@ -301,8 +302,9 @@ so the entire inner ``multipart/mixed`` is discarded.
X-Content-Filtered-By: Mailman/MimeDel ...
<BLANKLINE>
--AAA
+ Content-Transfer-Encoding: 7bit
MIME-Version: 1.0
- Content-Type: text/plain
+ Content-Type: text/plain; charset="us-ascii"
<BLANKLINE>
Converted text/html to text/plain
Filename: ...
=====================================
src/mailman/handlers/mime_delete.py
=====================================
@@ -274,7 +274,7 @@ def to_plaintext(msg):
resources.callback(shutil.rmtree, tempdir)
for subpart in typed_subpart_iterator(msg, 'text', 'html'):
filename = os.path.join(tempdir, '{}.html'.format(next(counter)))
- ctype = msg.get_content_charset('utf-8')
+ ctype = msg.get_content_charset('us-ascii')
with open(filename, 'w', encoding='utf-8') as fp:
fp.write(subpart.get_payload(decode=True).decode(ctype,
errors='replace'))
@@ -288,7 +288,7 @@ def to_plaintext(msg):
# Replace the payload of the subpart with the converted text
# and tweak the content type.
del subpart['content-transfer-encoding']
- subpart.set_payload(stdout)
+ subpart.set_payload(stdout, charset=ctype)
subpart.set_type('text/plain')
changedp += 1
return changedp
=====================================
src/mailman/handlers/tests/test_mimedel.py
=====================================
@@ -45,6 +45,9 @@ from zope.component import getUtility
@contextmanager
def dummy_script(arg=''):
exe = sys.executable
+ non_ascii = ''
+ if arg == 'non-ascii':
+ non_ascii = '‘...’'
extra = ''
if arg == 'scripterr':
extra = 'error'
@@ -59,10 +62,10 @@ def dummy_script(arg=''):
import sys
if len(sys.argv) > 2:
sys.exit(1)
-print('Converted text/html to text/plain')
+print('Converted text/html to text/plain{}')
print('Filename:', sys.argv[1])
print(open(sys.argv[1]).readlines()[0])
-""", file=fp)
+""".format(non_ascii), file=fp)
config.push('dummy script', """\
[mailman]
html_to_plain_text_command = {exe} {script} {extra} $filename
@@ -261,6 +264,29 @@ PGh0bWw+PGhlYWQ+PC9oZWFkPgo8Ym9keT48L2JvZHk+PC9odG1sPgo=
self.assertEqual(payload_lines[0], 'Converted text/html to text/plain')
self.assertEqual(payload_lines[2], '<html><head></head>')
+ def test_convert_html_to_plaintext_encodes_new_payload(self):
+ # Test that the converted payload with non-ascii is encoded.
+ msg = mfs("""\
+From: [email protected]
+Content-Type: text/html; charset=utf-8
+Content-Transfer-Encoding: base64
+MIME-Version: 1.0
+
+Q29udmVydGVkIHRleHQvaHRtbCB0byB0ZXh0L3BsYWlu4oCYLi4u4oCZCg==
+""")
+ process = config.handlers['mime-delete'].process
+ with dummy_script('non-ascii'):
+ process(self._mlist, msg, {})
+ self.assertEqual(msg['content-type'], 'text/plain; charset="utf-8"')
+ self.assertEqual(msg['content-transfer-encoding'], 'base64')
+ self.assertTrue(
+ msg['x-content-filtered-by'].startswith('Mailman/MimeDel'))
+ payload_lines = (
+ msg.get_payload(decode=True).decode('utf-8').splitlines())
+ self.assertEqual(payload_lines[0],
+ 'Converted text/html to text/plain‘...’')
+ self.assertTrue(payload_lines[1].startswith('Filename'))
+
def test_convert_html_to_plaintext_error_return(self):
# Calling a script which returns an error status is properly logged.
msg = mfs("""\
View it on GitLab:
https://gitlab.com/mailman/mailman/-/compare/fbec46157004dfbc3105670accff117d74614335...3af7ed0b5b98efb3a4203b8d7a81bea5f20bda46
--
View it on GitLab:
https://gitlab.com/mailman/mailman/-/compare/fbec46157004dfbc3105670accff117d74614335...3af7ed0b5b98efb3a4203b8d7a81bea5f20bda46
You're receiving this email because of your account on gitlab.com.
_______________________________________________
Mailman-checkins mailing list
[email protected]
Unsubscribe:
https://mail.python.org/mailman/options/mailman-checkins/archive%40jab.org
