https://github.com/python/cpython/commit/b25122e151d79faadf9f6ab385a89b5b949da286 commit: b25122e151d79faadf9f6ab385a89b5b949da286 branch: 3.15 author: Miss Islington (bot) <[email protected]> committer: JelleZijlstra <[email protected]> date: 2026-10-22T08:26:44-07:00 summary:
[3.15] gh-159739: Preserve non-BMP characters in multibyte encoders (GH-159987) (#160094) (cherry picked from commit 600f99da42b4c821ea908997771f4d5c343a37fb) Co-authored-by: Jelle Zijlstra <[email protected]> files: A Misc/NEWS.d/next/Library/2026-10-20-00-00-00.gh-issue-159739.3660a6.rst M Lib/test/test_multibytecodec.py M Modules/cjkcodecs/_codecs_hk.c M Modules/cjkcodecs/_codecs_iso2022.c M Modules/cjkcodecs/cjkcodecs.h diff --git a/Lib/test/test_multibytecodec.py b/Lib/test/test_multibytecodec.py index 0cd822fef874003..0b34c6e69fab6b6 100644 --- a/Lib/test/test_multibytecodec.py +++ b/Lib/test/test_multibytecodec.py @@ -43,6 +43,18 @@ def test_str_decode(self): for enc in ALL_CJKENCODINGS: self.assertEqual('abcd'.encode(enc), b'abcd') + def test_nonbmp_pair_modifier(self): + cases = [('big5hkscs', '\u00ca', '\u0304')] + cases.extend((encoding, '\u304b', '\u309a') for encoding in ( + 'euc_jis_2004', 'euc_jisx0213', 'shift_jis_2004', 'shift_jisx0213', + 'iso2022_jp_2004', 'iso2022_jp_3')) + for encoding, base, modifier in cases: + with self.subTest(encoding=encoding): + pair = base + modifier + self.assertEqual(pair.encode(encoding).decode(encoding), pair) + nonbmp = base + chr(0x10000 + ord(modifier)) + self.assertRaises(UnicodeEncodeError, nonbmp.encode, encoding) + def test_errorcallback_longindex(self): dec = codecs.getdecoder('euc-kr') myreplace = lambda exc: ('', sys.maxsize+1) diff --git a/Misc/NEWS.d/next/Library/2026-10-20-00-00-00.gh-issue-159739.3660a6.rst b/Misc/NEWS.d/next/Library/2026-10-20-00-00-00.gh-issue-159739.3660a6.rst new file mode 100644 index 000000000000000..56d11c767319cc3 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-10-20-00-00-00.gh-issue-159739.3660a6.rst @@ -0,0 +1,2 @@ +Fix multibyte encoders silently consuming a non-BMP character when looking up +a two-character mapping. diff --git a/Modules/cjkcodecs/_codecs_hk.c b/Modules/cjkcodecs/_codecs_hk.c index e7273bf18e34941..0e0c5670238555b 100644 --- a/Modules/cjkcodecs/_codecs_hk.c +++ b/Modules/cjkcodecs/_codecs_hk.c @@ -62,7 +62,7 @@ ENCODER(big5hkscs) else c2 = 0; - if (inlen - *inpos >= 2 && + if (inlen - *inpos >= 2 && c2 <= 0xffff && ((c & 0xffdf) == 0x00ca) && ((c2 & 0xfff7) == 0x0304)) { code = big5hkscs_pairenc_table[ diff --git a/Modules/cjkcodecs/_codecs_iso2022.c b/Modules/cjkcodecs/_codecs_iso2022.c index e79e6e768445552..662c2e1188e817c 100644 --- a/Modules/cjkcodecs/_codecs_iso2022.c +++ b/Modules/cjkcodecs/_codecs_iso2022.c @@ -804,7 +804,7 @@ jisx0213_encoder(const MultibyteCodec *codec, const Py_UCS4 *data, case 2: /* second character of unicode pair */ if (data[1] != 0) { /* Don't consume null char as part of pair */ - coded = find_pairencmap((ucs2_t)data[0], (ucs2_t)data[1], + coded = find_pairencmap((ucs2_t)data[0], data[1], jisx0213_pair_encmap, JISX0213_ENCPAIRS); if (coded != DBCINV) { return coded; diff --git a/Modules/cjkcodecs/cjkcodecs.h b/Modules/cjkcodecs/cjkcodecs.h index 9d86396f73b2b55..020559ac014b1f4 100644 --- a/Modules/cjkcodecs/cjkcodecs.h +++ b/Modules/cjkcodecs/cjkcodecs.h @@ -406,10 +406,13 @@ register_maps(PyObject *module) #ifdef USING_BINARY_PAIR_SEARCH static DBCHAR -find_pairencmap(ucs2_t body, ucs2_t modifier, +find_pairencmap(ucs2_t body, Py_UCS4 modifier, const struct pair_encodemap *haystack, int haystacksize) { int pos, min, max; + if (modifier > 0xffff) { + return DBCINV; + } Py_UCS4 value = body << 16 | modifier; min = 0; _______________________________________________ Python-checkins mailing list -- [email protected] To unsubscribe send an email to [email protected] https://mail.python.org/mailman3//lists/python-checkins.python.org Member address: [email protected]
