https://github.com/python/cpython/commit/709041c34b8bb8d04e63987348ea6f061fb99531 commit: 709041c34b8bb8d04e63987348ea6f061fb99531 branch: 3.14 author: Miss Islington (bot) <[email protected]> committer: serhiy-storchaka <[email protected]> date: 2026-08-04T17:13:24Z summary:
[3.14] gh-117807: Handle invalid UTF-8 in mimetypes map files (GH-151216) (GH-155184) (cherry picked from commit 30ac23e3e413c2d38b80691b1c80e3cd74a22639) Co-authored-by: Harjoth Khara <[email protected]> files: A Misc/NEWS.d/next/Library/2026-06-10-00-00-01.gh-issue-117807.Cx1178.rst M Lib/mimetypes.py M Lib/test/test_mimetypes.py diff --git a/Lib/mimetypes.py b/Lib/mimetypes.py index 7d0f4c1fd400d5..e7581d2ae0ace4 100644 --- a/Lib/mimetypes.py +++ b/Lib/mimetypes.py @@ -244,7 +244,7 @@ def read(self, filename, strict=True): list of standard types, else to the list of non-standard types. """ - with open(filename, encoding='utf-8') as fp: + with open(filename, encoding='utf-8', errors='surrogateescape') as fp: self.readfp(fp, strict) def readfp(self, fp, strict=True): @@ -435,7 +435,7 @@ def init(files=None): def read_mime_types(file): try: - f = open(file, encoding='utf-8') + f = open(file, encoding='utf-8', errors='surrogateescape') except OSError: return None with f: diff --git a/Lib/test/test_mimetypes.py b/Lib/test/test_mimetypes.py index a815ef6a24c1a5..a7d1d724ff0e18 100644 --- a/Lib/test/test_mimetypes.py +++ b/Lib/test/test_mimetypes.py @@ -67,9 +67,50 @@ def test_read_mime_types(self): with unittest.mock.patch.object(mimetypes, 'open', return_value=fp) as mock_open: mime_dict = mimetypes.read_mime_types(filename) - mock_open.assert_called_with(filename, encoding='utf-8') + mock_open.assert_called_with(filename, encoding='utf-8', + errors='surrogateescape') eq(mime_dict[".Français"], "application/no-mans-land") + def test_read_mime_types_invalid_utf8_comment(self): + with os_helper.temp_dir() as directory: + data = (b"# non-UTF-8 comment: \x83\n" + b"x-application/x-unittest pyunit\n") + file = os.path.join(directory, "sample.mimetype") + with open(file, "wb") as f: + f.write(data) + + mime_dict = mimetypes.read_mime_types(file) + self.assertEqual( + mime_dict[".pyunit"], "x-application/x-unittest") + + db = mimetypes.MimeTypes() + db.read(file) + self.assertEqual( + db.guess_file_type("sample.pyunit")[0], + "x-application/x-unittest") + + mimetypes.init(files=[file]) + self.assertEqual( + mimetypes.guess_file_type("sample.pyunit")[0], + "x-application/x-unittest") + + def test_read_mime_types_invalid_utf8_type(self): + # A non-UTF-8 byte in a type or extension (not only in a comment) is + # preserved via surrogateescape, so the mapping is not corrupted. + with os_helper.temp_dir() as directory: + data = (b"x-application/x-unittest pyunit\n" + b"application/bad\x83 badext\x83\n") + file = os.path.join(directory, "sample.mimetype") + with open(file, "wb") as f: + f.write(data) + + bad_type = b"application/bad\x83".decode("utf-8", "surrogateescape") + bad_ext = b".badext\x83".decode("utf-8", "surrogateescape") + + mime_dict = mimetypes.read_mime_types(file) + self.assertEqual(mime_dict[".pyunit"], "x-application/x-unittest") + self.assertEqual(mime_dict[bad_ext], bad_type) + def test_init_reinitializes(self): # Issue 4936: make sure an init starts clean # First, put some poison into the types table diff --git a/Misc/NEWS.d/next/Library/2026-06-10-00-00-01.gh-issue-117807.Cx1178.rst b/Misc/NEWS.d/next/Library/2026-06-10-00-00-01.gh-issue-117807.Cx1178.rst new file mode 100644 index 00000000000000..d6a874a84867a2 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-06-10-00-00-01.gh-issue-117807.Cx1178.rst @@ -0,0 +1,2 @@ +Fix :mod:`mimetypes` initialization from MIME map files containing invalid +UTF-8 bytes. _______________________________________________ Python-checkins mailing list -- [email protected] To unsubscribe send an email to [email protected] https://mail.python.org/mailman3//lists/python-checkins.python.org Member address: [email protected]
