https://github.com/python/cpython/commit/5aafbea2c9890ad8d179432a0ea2e06c6ca9cec3 commit: 5aafbea2c9890ad8d179432a0ea2e06c6ca9cec3 branch: main author: Serhiy Storchaka <[email protected]> committer: serhiy-storchaka <[email protected]> date: 2026-07-10T18:42:07+03:00 summary:
gh-153494: Encode imaplib search criteria to the declared CHARSET (GH-153495) IMAP4.search(), sort(), thread() and the uid SORT/THREAD variants now encode str search criteria to the declared charset, so international search text can be passed as ordinary str. A criterion passed as bytes is sent unchanged, for use with a charset that Python has no codec for. Co-Authored-By: Claude Opus 4.8 <[email protected]> files: A Misc/NEWS.d/next/Library/2026-07-10-16-00-00.gh-issue-153494.qCh8rT.rst M Doc/library/imaplib.rst M Doc/whatsnew/3.16.rst M Lib/imaplib.py M Lib/test/test_imaplib.py diff --git a/Doc/library/imaplib.rst b/Doc/library/imaplib.rst index 3f26243f77dd5a..7b79d81790b667 100644 --- a/Doc/library/imaplib.rst +++ b/Doc/library/imaplib.rst @@ -610,6 +610,13 @@ An :class:`IMAP4` instance has the following methods: If *uid* is true, the message numbers in the response are UIDs (``UID SEARCH``). + A criterion passed as :class:`str` is encoded to *charset* + (which must name a codec known to Python); + pass :class:`bytes` to send a criterion that is already encoded, + for example when *charset* is one that Python does not support. + When *charset* is ``None`` (as it must be under ``UTF8=ACCEPT``), + the criterion is sent using the connection's encoding instead. + Example:: # M is a connected IMAP4 instance... @@ -621,6 +628,9 @@ An :class:`IMAP4` instance has the following methods: .. versionchanged:: next Added the *uid* parameter. + .. versionchanged:: next + ``str`` search criteria are encoded to *charset*. + .. method:: IMAP4.select(mailbox='INBOX', readonly=False) @@ -682,11 +692,18 @@ An :class:`IMAP4` instance has the following methods: If *uid* is true, the message numbers in the response are UIDs (``UID SORT``). + As with :meth:`search`, + a *search_criterion* passed as :class:`str` is encoded to *charset*; + pass :class:`bytes` to send one already encoded. + This is an ``IMAP4rev1`` extension command. .. versionchanged:: next Added the *uid* parameter. + .. versionchanged:: next + ``str`` search criteria are encoded to *charset*. + .. method:: IMAP4.starttls(ssl_context=None) @@ -772,11 +789,18 @@ An :class:`IMAP4` instance has the following methods: If *uid* is true, the message numbers in the response are UIDs (``UID THREAD``). + As with :meth:`search`, + a *search_criterion* passed as :class:`str` is encoded to *charset*; + pass :class:`bytes` to send one already encoded. + This is an ``IMAP4rev1`` extension command. .. versionchanged:: next Added the *uid* parameter. + .. versionchanged:: next + ``str`` search criteria are encoded to *charset*. + .. method:: IMAP4.uid(command, arg[, ...]) diff --git a/Doc/whatsnew/3.16.rst b/Doc/whatsnew/3.16.rst index 9e1070e8efdaa5..06c3dbb2f0f1ad 100644 --- a/Doc/whatsnew/3.16.rst +++ b/Doc/whatsnew/3.16.rst @@ -276,6 +276,14 @@ imaplib :meth:`~imaplib.IMAP4.uid`. (Contributed by Serhiy Storchaka in :gh:`153502`.) +* :meth:`~imaplib.IMAP4.search`, :meth:`~imaplib.IMAP4.sort` + and :meth:`~imaplib.IMAP4.thread` (and the corresponding ``uid`` commands) + now encode :class:`str` search criteria to the declared *charset*, + so international search text can be passed as an ordinary :class:`str`. + When *charset* is ``None`` (as it must be under ``UTF8=ACCEPT``), + the criteria are sent using the connection encoding instead. + (Contributed by Serhiy Storchaka in :gh:`153494`.) + ipaddress --------- diff --git a/Lib/imaplib.py b/Lib/imaplib.py index 4e7619b61e89cd..ed5b528976448a 100644 --- a/Lib/imaplib.py +++ b/Lib/imaplib.py @@ -950,11 +950,15 @@ def search(self, charset, *criteria, uid=False): If UTF8 is enabled, charset MUST be None. If 'uid' is true, the message numbers in the response are UIDs (UID SEARCH). + + A 'criteria' passed as str is encoded to 'charset'; pass bytes to + send criteria that are already encoded. """ name = 'SEARCH' if charset is not None: if self.utf8_enabled: raise IMAP4.error("Non-None charset not valid in UTF8 mode") + criteria = self._encode_criteria(charset, criteria) args = ('CHARSET', self._astring(charset), *criteria) else: args = criteria @@ -1036,6 +1040,7 @@ def sort(self, sort_criteria, charset, *search_criteria, uid=False): #if not name in self.capabilities: # Let the server decide! # raise self.error('unimplemented extension command: %s' % name) sort_criteria = self._set_quote(sort_criteria) + search_criteria = self._encode_criteria(charset, search_criteria) if charset is not None: charset = self._astring(charset) args = (sort_criteria, charset, *search_criteria) @@ -1117,6 +1122,7 @@ def thread(self, threading_algorithm, charset, *search_criteria, uid=False): (UID THREAD). """ name = 'THREAD' + search_criteria = self._encode_criteria(charset, search_criteria) if charset is not None: charset = self._astring(charset) args = (self._atom(threading_algorithm), charset, *search_criteria) @@ -1158,12 +1164,14 @@ def uid(self, command, *args): self._set_quote(flags)) elif command == 'SORT': sort_criteria, charset, *search_criteria = args + search_criteria = self._encode_criteria(charset, search_criteria) if charset is not None: charset = self._astring(charset) args = (self._set_quote(sort_criteria), charset, *search_criteria) elif command == 'THREAD': threading_algorithm, charset, *search_criteria = args + search_criteria = self._encode_criteria(charset, search_criteria) if charset is not None: charset = self._astring(charset) args = (self._atom(threading_algorithm), charset, @@ -1576,6 +1584,17 @@ def _fetch_parts(self, arg): return arg return self._set_quote(arg) + def _encode_criteria(self, charset, criteria): + # Encode str search criteria to the declared CHARSET so the bytes on + # the wire match it. bytes criteria are already encoded and pass + # through unchanged. charset is None when no CHARSET is sent. + if charset is None: + return criteria + if isinstance(charset, (bytes, bytearray)): + charset = str(charset, 'ascii') + return tuple(c.encode(charset) if isinstance(c, str) else c + for c in criteria) + def _quote(self, arg): if isinstance(arg, str): arg = bytes(arg, self._encoding) diff --git a/Lib/test/test_imaplib.py b/Lib/test/test_imaplib.py index 2413a8728ec72b..3eaca67299e7c3 100644 --- a/Lib/test/test_imaplib.py +++ b/Lib/test/test_imaplib.py @@ -205,6 +205,32 @@ def test_astring(self): self.assertEqual(m._astring('Entwürfe'), '"Entwürfe"'.encode()) self.assertEqual(m._astring(b'Entw\xc3\xbcrfe'), b'"Entw\xc3\xbcrfe"') + def test_encode_criteria(self): + m = imaplib.IMAP4.__new__(imaplib.IMAP4) + enc = m._encode_criteria + # No charset: criteria are returned unchanged. + self.assertEqual(enc(None, ('TEXT', 'x')), ('TEXT', 'x')) + # str criteria are encoded to the charset; ASCII is charset-independent. + self.assertEqual(enc('UTF-8', ('TEXT', 'XXXXXX')), (b'TEXT', b'XXXXXX')) + # Non-ASCII text is encoded to the declared charset, including charsets + # other than ASCII, Latin-1 and UTF-8. + self.assertEqual(enc('UTF-8', ('"café"',)), ('"café"'.encode('utf-8'),)) + self.assertEqual(enc('ISO-8859-1', ('"café"',)), + ('"café"'.encode('latin-1'),)) + self.assertEqual(enc('KOI8-U', ('"Київ"',)), + ('"Київ"'.encode('koi8-u'),)) + self.assertEqual(enc('SHIFT_JIS', ('"日本"',)), + ('"日本"'.encode('shift_jis'),)) + # bytes criteria are already encoded and pass through unchanged. + self.assertEqual(enc('SHIFT_JIS', (b'"already"',)), (b'"already"',)) + # The charset name may itself be bytes. + self.assertEqual(enc(b'UTF-8', ('"café"',)), ('"café"'.encode('utf-8'),)) + # A charset with no codec at all (not even via the iconv codec) cannot + # encode str criteria; bytes criteria must be used with such + # server-only charsets. + self.assertRaises(LookupError, enc, 'no-such-charset', ('TEXT',)) + self.assertEqual(enc('no-such-charset', (b'TEXT',)), (b'TEXT',)) + def test_astring_idempotent(self): # Quoting an already quoted argument should not change it, so that # quoting twice gives the same result as quoting once. @@ -337,7 +363,11 @@ def handle(self): except StopIteration: self.continuation = None continue - splitline = splitargs(line.decode().removesuffix('\r\n')) + self.server.line = line + # surrogateescape so a criterion encoded in a non-UTF-8 charset + # does not crash the handler; tests inspect server.line for bytes. + splitline = splitargs(line.decode('utf-8', 'surrogateescape') + .removesuffix('\r\n')) tag = splitline[0] cmd = splitline[1] args = splitline[2:] @@ -1546,7 +1576,17 @@ def test_search(self): self.assertEqual(data, [b'43']) self.assertEqual(server.args, ['CHARSET', 'UTF-8', 'TEXT', 'XXXXXX']) - typ, data = client.search('NF_Z_62-010_(1973)', 'TEXT', 'XXXXXX') + # A non-ASCII str criterion is encoded to the declared charset (KOI8-U + # here, which is not UTF-8, so check the encoded bytes on the wire). + response[:] = ['* SEARCH 43'] + typ, data = client.search('KOI8-U', 'SUBJECT', '"Київ"') + self.assertEqual(typ, 'OK') + self.assertIn(b'CHARSET KOI8-U ', server.line) + self.assertIn('"Київ"'.encode('koi8-u'), server.line) + + # bytes criteria keep this focused on charset-name quoting (the + # parentheses force the name to be quoted) without criteria encoding. + typ, data = client.search('NF_Z_62-010_(1973)', b'TEXT', b'XXXXXX') self.assertEqual(typ, 'OK') self.assertEqual(server.args, ['CHARSET', '"NF_Z_62-010_(1973)"', 'TEXT', 'XXXXXX']) @@ -1616,10 +1656,16 @@ def test_sort(self): self.assertEqual(data, [br'']) self.assertEqual(server.args, ['(SUBJECT)', 'US-ASCII', 'TEXT', '"not in mailbox"']) - typ, data = client.sort('SUBJECT', 'NF_Z_62-010_(1973)', 'TEXT', '"not in mailbox"') + typ, data = client.sort('SUBJECT', 'NF_Z_62-010_(1973)', b'TEXT', b'"not in mailbox"') self.assertEqual(typ, 'OK') self.assertEqual(server.args, ['(SUBJECT)', '"NF_Z_62-010_(1973)"', 'TEXT', '"not in mailbox"']) + # A non-ASCII str criterion is encoded to the declared charset. + response[:] = ['* SORT'] + typ, data = client.sort('(SUBJECT)', 'KOI8-U', 'TEXT', '"Київ"') + self.assertEqual(typ, 'OK') + self.assertIn('"Київ"'.encode('koi8-u'), server.line) + def test_uid_sort(self): response = [] client, server = self._setup(make_simple_handler('UID', response, @@ -1644,10 +1690,16 @@ def test_uid_sort(self): self.assertEqual(data, [br'']) self.assertEqual(server.args, ['SORT', '(SUBJECT)', 'US-ASCII', 'TEXT', '"not in mailbox"']) - typ, data = client.uid('sort', 'SUBJECT', 'NF_Z_62-010_(1973)', 'TEXT', '"not in mailbox"') + typ, data = client.uid('sort', 'SUBJECT', 'NF_Z_62-010_(1973)', b'TEXT', b'"not in mailbox"') self.assertEqual(typ, 'OK') self.assertEqual(server.args, ['SORT', '(SUBJECT)', '"NF_Z_62-010_(1973)"', 'TEXT', '"not in mailbox"']) + # A non-ASCII str criterion is encoded to the declared charset. + response[:] = ['* SORT'] + typ, data = client.uid('sort', '(SUBJECT)', 'KOI8-U', 'TEXT', '"Київ"') + self.assertEqual(typ, 'OK') + self.assertIn('"Київ"'.encode('koi8-u'), server.line) + # The uid=True keyword is a shorthand for uid('SORT', ...). response[:] = ['* SORT 2 84 882'] typ, data = client.sort('(SUBJECT)', 'UTF-8', 'SINCE', '1-Feb-1994', uid=True) @@ -1692,10 +1744,16 @@ def test_thread(self): b'(199)(200 202)(201)(203)(204)(205 206 207)(208)']) self.assertEqual(server.args, ['ORDEREDSUBJECT', 'US-ASCII', 'TEXT', '"gewp"']) - typ, data = client.thread('ORDEREDSUBJECT', 'NF_Z_62-010_(1973)', 'TEXT', '"gewp"') + typ, data = client.thread('ORDEREDSUBJECT', 'NF_Z_62-010_(1973)', b'TEXT', b'"gewp"') self.assertEqual(typ, 'OK') self.assertEqual(server.args, ['ORDEREDSUBJECT', '"NF_Z_62-010_(1973)"', 'TEXT', '"gewp"']) + # A non-ASCII str criterion is encoded to the declared charset. + response[:] = ['* THREAD (1)'] + typ, data = client.thread('ORDEREDSUBJECT', 'KOI8-U', 'TEXT', '"Київ"') + self.assertEqual(typ, 'OK') + self.assertIn('"Київ"'.encode('koi8-u'), server.line) + def test_uid_thread(self): response = [] client, server = self._setup(make_simple_handler('UID', response, @@ -1734,10 +1792,16 @@ def test_uid_thread(self): b'(199)(200 202)(201)(203)(204)(205 206 207)(208)']) self.assertEqual(server.args, ['THREAD', 'ORDEREDSUBJECT', 'US-ASCII', 'TEXT', '"gewp"']) - typ, data = client.uid('THREAD', 'ORDEREDSUBJECT', 'NF_Z_62-010_(1973)', 'TEXT', '"gewp"') + typ, data = client.uid('THREAD', 'ORDEREDSUBJECT', 'NF_Z_62-010_(1973)', b'TEXT', b'"gewp"') self.assertEqual(typ, 'OK') self.assertEqual(server.args, ['THREAD', 'ORDEREDSUBJECT', '"NF_Z_62-010_(1973)"', 'TEXT', '"gewp"']) + # A non-ASCII str criterion is encoded to the declared charset. + response[:] = ['* THREAD (1)'] + typ, data = client.uid('THREAD', 'ORDEREDSUBJECT', 'KOI8-U', 'TEXT', '"Київ"') + self.assertEqual(typ, 'OK') + self.assertIn('"Київ"'.encode('koi8-u'), server.line) + # The uid=True keyword is a shorthand for uid('THREAD', ...). response[:] = ['* THREAD (166)(167)(168)'] typ, data = client.thread('ORDEREDSUBJECT', 'UTF-8', 'SINCE', '5-MAR-2000', diff --git a/Misc/NEWS.d/next/Library/2026-07-10-16-00-00.gh-issue-153494.qCh8rT.rst b/Misc/NEWS.d/next/Library/2026-07-10-16-00-00.gh-issue-153494.qCh8rT.rst new file mode 100644 index 00000000000000..6e83940efcc180 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-10-16-00-00.gh-issue-153494.qCh8rT.rst @@ -0,0 +1,8 @@ +:meth:`imaplib.IMAP4.search`, :meth:`~imaplib.IMAP4.sort` +and :meth:`~imaplib.IMAP4.thread` (and the corresponding ``uid`` commands) +now encode :class:`str` search criteria to the declared *charset*, +so international search text can be passed as ordinary :class:`str`. +When *charset* is ``None`` (as it must be under ``UTF8=ACCEPT``), +the criteria are sent using the connection encoding instead. +A criterion passed as :class:`bytes` is sent unchanged, +for use with a charset that Python has no codec for. _______________________________________________ Python-checkins mailing list -- [email protected] To unsubscribe send an email to [email protected] https://mail.python.org/mailman3//lists/python-checkins.python.org Member address: [email protected]
