Serhiy Storchaka <storch...@gmail.com> added the comment:
Here is updated patch with resolved merge conflict with 3214c9ebcf5e.
----------
Added file: http://bugs.python.org/file26118/issue8271-3.3-fast-3.patch
_______________________________________
Python tracker <rep...@bugs.python.org>
<http://bugs.python.org/issue8271>
_______________________________________
diff -r ec656d79b8ac Lib/test/test_unicode.py
--- a/Lib/test/test_unicode.py Sat Jun 23 22:55:58 2012 +0200
+++ b/Lib/test/test_unicode.py Sun Jun 24 00:20:16 2012 +0300
@@ -1321,7 +1321,7 @@
# with start byte of a 2-byte sequence
(b'\xc2', FFFD), # only the start byte
(b'\xc2\xc2', FFFD*2), # 2 start bytes
- (b'\xc2\xc2\xc2', FFFD*3), # 2 start bytes
+ (b'\xc2\xc2\xc2', FFFD*3), # 3 start bytes
(b'\xc2\x41', FFFD+'A'), # invalid continuation byte
# with start byte of a 3-byte sequence
(b'\xe1', FFFD), # only the start byte
@@ -1391,6 +1391,233 @@
self.assertEqual(seq.decode('utf-8', 'ignore'),
res.replace('\uFFFD', ''))
+ def to_bytestring(self, seq):
+ return bytes(int(c, 16) for c in seq.split())
+
+ def assertCorrectUTF8Decoding(self, seq, res, err):
+ """
+ Check that an invalid UTF-8 sequence raises an UnicodeDecodeError when
+ 'strict' is used, returns res when 'replace' is used, and that doesn't
+ return anything when 'ignore' is used.
+ """
+ #with self.assertRaises(UnicodeDecodeError) as cm:
+ #seq.decode('utf-8')
+ #exc = cm.exception
+ try:
+ seq.decode('utf-8')
+ except UnicodeDecodeError as e:
+ exc = str(e)
+ else:
+ self.fail(
+ "%r.decode('utf-8') didn't raise UnicodeDecodeError." % seq)
+
+ self.assertIn(err, str(exc))
+ self.assertEqual(seq.decode('utf-8', 'replace'), res)
+ self.assertEqual((b'aaaa' + seq + b'bbbb').decode('utf-8', 'replace'),
+ 'aaaa' + res + 'bbbb')
+ res = res.replace('\ufffd', '')
+ self.assertEqual(seq.decode('utf-8', 'ignore'), res)
+ self.assertEqual((b'aaaa' + seq + b'bbbb').decode('utf-8', 'ignore'),
+ 'aaaa' + res + 'bbbb')
+
+ def test_invalid_start_byte(self):
+ """
+ Test that an 'invalid start byte' error is raised when the first byte
+ is not in the ASCII range or is not a valid start byte of a 2-, 3-, or
+ 4-bytes sequence. The invalid start byte is replaced with a single
+ U+FFFD when errors='replace'.
+ E.g. <80> is a continuation byte and can appear only after a start
byte.
+ """
+ FFFD = '\ufffd'
+ for byte in b'\x80\xA0\x9F\xBF\xC0\xC1\xF5\xFF':
+ self.assertCorrectUTF8Decoding(bytes([byte]), '\ufffd',
+ 'invalid start byte')
+
+ def test_unexpected_end_of_data(self):
+ """
+ Test that an 'unexpected end of data' error is raised when the string
+ ends after a start byte of a 2-, 3-, or 4-bytes sequence without having
+ enough continuation bytes. The incomplete sequence is replaced with a
+ single U+FFFD when errors='replace'.
+ E.g. in the sequence <F3 80 80>, F3 is the start byte of a 4-bytes
+ sequence, but it's followed by only 2 valid continuation bytes and the
+ last continuation bytes is missing.
+ Note: the continuation bytes must be all valid, if one of them is
+ invalid another error will be raised.
+ """
+ sequences = [
+ 'C2', 'DF',
+ 'E0 A0', 'E0 BF', 'E1 80', 'E1 BF', 'EC 80', 'EC BF',
+ 'ED 80', 'ED 9F', 'EE 80', 'EE BF', 'EF 80', 'EF BF',
+ 'F0 90', 'F0 BF', 'F0 90 80', 'F0 90 BF', 'F0 BF 80', 'F0 BF BF',
+ 'F1 80', 'F1 BF', 'F1 80 80', 'F1 80 BF', 'F1 BF 80', 'F1 BF BF',
+ 'F3 80', 'F3 BF', 'F3 80 80', 'F3 80 BF', 'F3 BF 80', 'F3 BF BF',
+ 'F4 80', 'F4 8F', 'F4 80 80', 'F4 80 BF', 'F4 8F 80', 'F4 8F BF'
+ ]
+ FFFD = '\ufffd'
+ for seq in sequences:
+ self.assertCorrectUTF8Decoding(self.to_bytestring(seq), '\ufffd',
+ 'unexpected end of data')
+
+ def test_invalid_cb_for_2bytes_seq(self):
+ """
+ Test that an 'invalid continuation byte' error is raised when the
+ continuation byte of a 2-bytes sequence is invalid. The start byte
+ is replaced by a single U+FFFD and the second byte is handled
+ separately when errors='replace'.
+ E.g. in the sequence <C2 41>, C2 is the start byte of a 2-bytes
+ sequence, but 41 is not a valid continuation byte because it's the
+ ASCII letter 'A'.
+ """
+ FFFD = '\ufffd'
+ FFFDx2 = FFFD * 2
+ sequences = [
+ ('C2 00', FFFD+'\x00'), ('C2 7F', FFFD+'\x7f'),
+ ('C2 C0', FFFDx2), ('C2 FF', FFFDx2),
+ ('DF 00', FFFD+'\x00'), ('DF 7F', FFFD+'\x7f'),
+ ('DF C0', FFFDx2), ('DF FF', FFFDx2),
+ ]
+ for seq, res in sequences:
+ self.assertCorrectUTF8Decoding(self.to_bytestring(seq), res,
+ 'invalid continuation byte')
+
+ def test_invalid_cb_for_3bytes_seq(self):
+ """
+ Test that an 'invalid continuation byte' error is raised when the
+ continuation byte(s) of a 3-bytes sequence are invalid. When
+ errors='replace', if the first continuation byte is valid, the first
+ two bytes (start byte + 1st cb) are replaced by a single U+FFFD and the
+ third byte is handled separately, otherwise only the start byte is
+ replaced with a U+FFFD and the other continuation bytes are handled
+ separately.
+ E.g. in the sequence <E1 80 41>, E1 is the start byte of a 3-bytes
+ sequence, 80 is a valid continuation byte, but 41 is not a valid cb
+ because it's the ASCII letter 'A'.
+ Note: when the start byte is E0 or ED, the valid ranges for the first
+ continuation byte are limited to A0..BF and 80..9F respectively.
+ However, when the start byte is ED, Python 2 considers all the bytes
+ in range 80..BF valid. This is fixed in Python 3.
+ """
+ FFFD = '\ufffd'
+ FFFDx2 = FFFD * 2
+ sequences = [
+ ('E0 00', FFFD+'\x00'), ('E0 7F', FFFD+'\x7f'), ('E0 80', FFFDx2),
+ ('E0 9F', FFFDx2), ('E0 C0', FFFDx2), ('E0 FF', FFFDx2),
+ ('E0 A0 00', FFFD+'\x00'), ('E0 A0 7F', FFFD+'\x7f'),
+ ('E0 A0 C0', FFFDx2), ('E0 A0 FF', FFFDx2),
+ ('E0 BF 00', FFFD+'\x00'), ('E0 BF 7F', FFFD+'\x7f'),
+ ('E0 BF C0', FFFDx2), ('E0 BF FF', FFFDx2), ('E1 00', FFFD+'\x00'),
+ ('E1 7F', FFFD+'\x7f'), ('E1 C0', FFFDx2), ('E1 FF', FFFDx2),
+ ('E1 80 00', FFFD+'\x00'), ('E1 80 7F', FFFD+'\x7f'),
+ ('E1 80 C0', FFFDx2), ('E1 80 FF', FFFDx2),
+ ('E1 BF 00', FFFD+'\x00'), ('E1 BF 7F', FFFD+'\x7f'),
+ ('E1 BF C0', FFFDx2), ('E1 BF FF', FFFDx2), ('EC 00', FFFD+'\x00'),
+ ('EC 7F', FFFD+'\x7f'), ('EC C0', FFFDx2), ('EC FF', FFFDx2),
+ ('EC 80 00', FFFD+'\x00'), ('EC 80 7F', FFFD+'\x7f'),
+ ('EC 80 C0', FFFDx2), ('EC 80 FF', FFFDx2),
+ ('EC BF 00', FFFD+'\x00'), ('EC BF 7F', FFFD+'\x7f'),
+ ('EC BF C0', FFFDx2), ('EC BF FF', FFFDx2), ('ED 00', FFFD+'\x00'),
+ ('ED 7F', FFFD+'\x7f'),
+ # ('ED A0', FFFDx2), ('ED BF', FFFDx2), # see note ^
+ ('ED C0', FFFDx2), ('ED FF', FFFDx2), ('ED 80 00', FFFD+'\x00'),
+ ('ED 80 7F', FFFD+'\x7f'), ('ED 80 C0', FFFDx2),
+ ('ED 80 FF', FFFDx2), ('ED 9F 00', FFFD+'\x00'),
+ ('ED 9F 7F', FFFD+'\x7f'), ('ED 9F C0', FFFDx2),
+ ('ED 9F FF', FFFDx2), ('EE 00', FFFD+'\x00'),
+ ('EE 7F', FFFD+'\x7f'), ('EE C0', FFFDx2), ('EE FF', FFFDx2),
+ ('EE 80 00', FFFD+'\x00'), ('EE 80 7F', FFFD+'\x7f'),
+ ('EE 80 C0', FFFDx2), ('EE 80 FF', FFFDx2),
+ ('EE BF 00', FFFD+'\x00'), ('EE BF 7F', FFFD+'\x7f'),
+ ('EE BF C0', FFFDx2), ('EE BF FF', FFFDx2), ('EF 00', FFFD+'\x00'),
+ ('EF 7F', FFFD+'\x7f'), ('EF C0', FFFDx2), ('EF FF', FFFDx2),
+ ('EF 80 00', FFFD+'\x00'), ('EF 80 7F', FFFD+'\x7f'),
+ ('EF 80 C0', FFFDx2), ('EF 80 FF', FFFDx2),
+ ('EF BF 00', FFFD+'\x00'), ('EF BF 7F', FFFD+'\x7f'),
+ ('EF BF C0', FFFDx2), ('EF BF FF', FFFDx2),
+ ]
+ for seq, res in sequences:
+ self.assertCorrectUTF8Decoding(self.to_bytestring(seq), res,
+ 'invalid continuation byte')
+
+ def test_invalid_cb_for_4bytes_seq(self):
+ """
+ Test that an 'invalid continuation byte' error is raised when the
+ continuation byte(s) of a 4-bytes sequence are invalid. When
+ errors='replace',the start byte and all the following valid
+ continuation bytes are replaced with a single U+FFFD, and all the bytes
+ starting from the first invalid continuation bytes (included) are
+ handled separately.
+ E.g. in the sequence <E1 80 41>, E1 is the start byte of a 3-bytes
+ sequence, 80 is a valid continuation byte, but 41 is not a valid cb
+ because it's the ASCII letter 'A'.
+ Note: when the start byte is E0 or ED, the valid ranges for the first
+ continuation byte are limited to A0..BF and 80..9F respectively.
+ However, when the start byte is ED, Python 2 considers all the bytes
+ in range 80..BF valid. This is fixed in Python 3.
+ """
+ FFFD = '\ufffd'
+ FFFDx2 = FFFD * 2
+ sequences = [
+ ('F0 00', FFFD+'\x00'), ('F0 7F', FFFD+'\x7f'), ('F0 80', FFFDx2),
+ ('F0 8F', FFFDx2), ('F0 C0', FFFDx2), ('F0 FF', FFFDx2),
+ ('F0 90 00', FFFD+'\x00'), ('F0 90 7F', FFFD+'\x7f'),
+ ('F0 90 C0', FFFDx2), ('F0 90 FF', FFFDx2),
+ ('F0 BF 00', FFFD+'\x00'), ('F0 BF 7F', FFFD+'\x7f'),
+ ('F0 BF C0', FFFDx2), ('F0 BF FF', FFFDx2),
+ ('F0 90 80 00', FFFD+'\x00'), ('F0 90 80 7F', FFFD+'\x7f'),
+ ('F0 90 80 C0', FFFDx2), ('F0 90 80 FF', FFFDx2),
+ ('F0 90 BF 00', FFFD+'\x00'), ('F0 90 BF 7F', FFFD+'\x7f'),
+ ('F0 90 BF C0', FFFDx2), ('F0 90 BF FF', FFFDx2),
+ ('F0 BF 80 00', FFFD+'\x00'), ('F0 BF 80 7F', FFFD+'\x7f'),
+ ('F0 BF 80 C0', FFFDx2), ('F0 BF 80 FF', FFFDx2),
+ ('F0 BF BF 00', FFFD+'\x00'), ('F0 BF BF 7F', FFFD+'\x7f'),
+ ('F0 BF BF C0', FFFDx2), ('F0 BF BF FF', FFFDx2),
+ ('F1 00', FFFD+'\x00'), ('F1 7F', FFFD+'\x7f'), ('F1 C0', FFFDx2),
+ ('F1 FF', FFFDx2), ('F1 80 00', FFFD+'\x00'),
+ ('F1 80 7F', FFFD+'\x7f'), ('F1 80 C0', FFFDx2),
+ ('F1 80 FF', FFFDx2), ('F1 BF 00', FFFD+'\x00'),
+ ('F1 BF 7F', FFFD+'\x7f'), ('F1 BF C0', FFFDx2),
+ ('F1 BF FF', FFFDx2), ('F1 80 80 00', FFFD+'\x00'),
+ ('F1 80 80 7F', FFFD+'\x7f'), ('F1 80 80 C0', FFFDx2),
+ ('F1 80 80 FF', FFFDx2), ('F1 80 BF 00', FFFD+'\x00'),
+ ('F1 80 BF 7F', FFFD+'\x7f'), ('F1 80 BF C0', FFFDx2),
+ ('F1 80 BF FF', FFFDx2), ('F1 BF 80 00', FFFD+'\x00'),
+ ('F1 BF 80 7F', FFFD+'\x7f'), ('F1 BF 80 C0', FFFDx2),
+ ('F1 BF 80 FF', FFFDx2), ('F1 BF BF 00', FFFD+'\x00'),
+ ('F1 BF BF 7F', FFFD+'\x7f'), ('F1 BF BF C0', FFFDx2),
+ ('F1 BF BF FF', FFFDx2), ('F3 00', FFFD+'\x00'),
+ ('F3 7F', FFFD+'\x7f'), ('F3 C0', FFFDx2), ('F3 FF', FFFDx2),
+ ('F3 80 00', FFFD+'\x00'), ('F3 80 7F', FFFD+'\x7f'),
+ ('F3 80 C0', FFFDx2), ('F3 80 FF', FFFDx2),
+ ('F3 BF 00', FFFD+'\x00'), ('F3 BF 7F', FFFD+'\x7f'),
+ ('F3 BF C0', FFFDx2), ('F3 BF FF', FFFDx2),
+ ('F3 80 80 00', FFFD+'\x00'), ('F3 80 80 7F', FFFD+'\x7f'),
+ ('F3 80 80 C0', FFFDx2), ('F3 80 80 FF', FFFDx2),
+ ('F3 80 BF 00', FFFD+'\x00'), ('F3 80 BF 7F', FFFD+'\x7f'),
+ ('F3 80 BF C0', FFFDx2), ('F3 80 BF FF', FFFDx2),
+ ('F3 BF 80 00', FFFD+'\x00'), ('F3 BF 80 7F', FFFD+'\x7f'),
+ ('F3 BF 80 C0', FFFDx2), ('F3 BF 80 FF', FFFDx2),
+ ('F3 BF BF 00', FFFD+'\x00'), ('F3 BF BF 7F', FFFD+'\x7f'),
+ ('F3 BF BF C0', FFFDx2), ('F3 BF BF FF', FFFDx2),
+ ('F4 00', FFFD+'\x00'), ('F4 7F', FFFD+'\x7f'), ('F4 90', FFFDx2),
+ ('F4 BF', FFFDx2), ('F4 C0', FFFDx2), ('F4 FF', FFFDx2),
+ ('F4 80 00', FFFD+'\x00'), ('F4 80 7F', FFFD+'\x7f'),
+ ('F4 80 C0', FFFDx2), ('F4 80 FF', FFFDx2),
+ ('F4 8F 00', FFFD+'\x00'), ('F4 8F 7F', FFFD+'\x7f'),
+ ('F4 8F C0', FFFDx2), ('F4 8F FF', FFFDx2),
+ ('F4 80 80 00', FFFD+'\x00'), ('F4 80 80 7F', FFFD+'\x7f'),
+ ('F4 80 80 C0', FFFDx2), ('F4 80 80 FF', FFFDx2),
+ ('F4 80 BF 00', FFFD+'\x00'), ('F4 80 BF 7F', FFFD+'\x7f'),
+ ('F4 80 BF C0', FFFDx2), ('F4 80 BF FF', FFFDx2),
+ ('F4 8F 80 00', FFFD+'\x00'), ('F4 8F 80 7F', FFFD+'\x7f'),
+ ('F4 8F 80 C0', FFFDx2), ('F4 8F 80 FF', FFFDx2),
+ ('F4 8F BF 00', FFFD+'\x00'), ('F4 8F BF 7F', FFFD+'\x7f'),
+ ('F4 8F BF C0', FFFDx2), ('F4 8F BF FF', FFFDx2)
+ ]
+ for seq, res in sequences:
+ self.assertCorrectUTF8Decoding(self.to_bytestring(seq), res,
+ 'invalid continuation byte')
+
def test_codecs_idna(self):
# Test whether trailing dot is preserved
self.assertEqual("www.python.org.".encode("idna"), b"www.python.org.")
diff -r ec656d79b8ac Objects/stringlib/codecs.h
--- a/Objects/stringlib/codecs.h Sat Jun 23 22:55:58 2012 +0200
+++ b/Objects/stringlib/codecs.h Sun Jun 24 00:20:16 2012 +0300
@@ -94,15 +94,14 @@
}
}
- if (ch < 0xC2) {
- /* invalid sequence
- \x80-\xBF -- continuation byte
- \xC0-\xC1 -- fake 0000-007F */
- goto InvalidStart;
- }
-
if (ch < 0xE0) {
/* \xC2\x80-\xDF\xBF -- 0080-07FF */
+ if (ch < 0xC2) {
+ /* invalid sequence
+ \x80-\xBF -- continuation byte
+ \xC0-\xC1 -- fake 0000-007F */
+ goto InvalidStart;
+ }
Py_UCS4 ch2;
if (end - s < 2) {
/* unexpected end of data: the caller will decide whether
@@ -112,14 +111,15 @@
ch2 = (unsigned char)s[1];
if (!IS_CONTINUATION_BYTE(ch2))
/* invalid continuation byte */
- goto InvalidContinuation;
+ goto InvalidContinuation1;
ch = (ch << 6) + ch2 -
((0xC0 << 6) + 0x80);
assert ((ch > 0x007F) && (ch <= 0x07FF));
s += 2;
if (STRINGLIB_MAX_CHAR <= 0x007F ||
(STRINGLIB_MAX_CHAR < 0x07FF && ch > STRINGLIB_MAX_CHAR))
- goto Overflow;
+ /* Out-of-range */
+ goto Return;
*p++ = ch;
continue;
}
@@ -130,28 +130,37 @@
if (end - s < 3) {
/* unexpected end of data: the caller will decide whether
it's an error or not */
+ if (end - s < 2)
+ break;
+ ch2 = (unsigned char)s[1];
+ if (!IS_CONTINUATION_BYTE(ch2) ||
+ (ch2 < 0xA0 ? ch == 0xE0 : ch == 0xED))
+ /* for clarification see comments below */
+ goto InvalidContinuation1;
break;
}
ch2 = (unsigned char)s[1];
ch3 = (unsigned char)s[2];
- if (!IS_CONTINUATION_BYTE(ch2) ||
- !IS_CONTINUATION_BYTE(ch3)) {
+ if (!IS_CONTINUATION_BYTE(ch2)) {
/* invalid continuation byte */
- goto InvalidContinuation;
+ goto InvalidContinuation1;
}
if (ch == 0xE0) {
if (ch2 < 0xA0)
/* invalid sequence
\xE0\x80\x80-\xE0\x9F\xBF -- fake 0000-0800 */
- goto InvalidContinuation;
- }
- else if (ch == 0xED && ch2 > 0x9F) {
+ goto InvalidContinuation1;
+ } else if (ch == 0xED && ch2 >= 0xA0) {
/* Decoding UTF-8 sequences in range \xED\xA0\x80-\xED\xBF\xBF
will result in surrogates in range D800-DFFF. Surrogates are
not valid UTF-8 so they are rejected.
See http://www.unicode.org/versions/Unicode5.2.0/ch03.pdf
(table 3-7) and http://www.rfc-editor.org/rfc/rfc3629.txt */
- goto InvalidContinuation;
+ goto InvalidContinuation1;
+ }
+ if (!IS_CONTINUATION_BYTE(ch3)) {
+ /* invalid continuation byte */
+ goto InvalidContinuation2;
}
ch = (ch << 12) + (ch2 << 6) + ch3 -
((0xE0 << 12) + (0x80 << 6) + 0x80);
@@ -159,7 +168,8 @@
s += 3;
if (STRINGLIB_MAX_CHAR <= 0x07FF ||
(STRINGLIB_MAX_CHAR < 0xFFFF && ch > STRINGLIB_MAX_CHAR))
- goto Overflow;
+ /* Out-of-range */
+ goto Return;
*p++ = ch;
continue;
}
@@ -170,27 +180,44 @@
if (end - s < 4) {
/* unexpected end of data: the caller will decide whether
it's an error or not */
+ if (end - s < 2)
+ break;
+ ch2 = (unsigned char)s[1];
+ if (!IS_CONTINUATION_BYTE(ch2) ||
+ (ch2 < 0x90 ? ch == 0xF0 : ch == 0xF4))
+ /* for clarification see comments below */
+ goto InvalidContinuation1;
+ if (end - s < 3)
+ break;
+ ch3 = (unsigned char)s[2];
+ if (!IS_CONTINUATION_BYTE(ch3))
+ goto InvalidContinuation2;
break;
}
ch2 = (unsigned char)s[1];
ch3 = (unsigned char)s[2];
ch4 = (unsigned char)s[3];
- if (!IS_CONTINUATION_BYTE(ch2) ||
- !IS_CONTINUATION_BYTE(ch3) ||
- !IS_CONTINUATION_BYTE(ch4)) {
+ if (!IS_CONTINUATION_BYTE(ch2)) {
/* invalid continuation byte */
- goto InvalidContinuation;
+ goto InvalidContinuation1;
}
if (ch == 0xF0) {
if (ch2 < 0x90)
/* invalid sequence
- \xF0\x80\x80\x80-\xF0\x80\xBF\xBF -- fake 0000-FFFF */
- goto InvalidContinuation;
- }
- else if (ch == 0xF4 && ch2 > 0x8F) {
+ \xF0\x80\x80\x80-\xF0\x8F\xBF\xBF -- fake 0000-FFFF */
+ goto InvalidContinuation1;
+ } else if (ch == 0xF4 && ch2 >= 0x90) {
/* invalid sequence
\xF4\x90\x80\80- -- 110000- overflow */
- goto InvalidContinuation;
+ goto InvalidContinuation1;
+ }
+ if (!IS_CONTINUATION_BYTE(ch3)) {
+ /* invalid continuation byte */
+ goto InvalidContinuation2;
+ }
+ if (!IS_CONTINUATION_BYTE(ch4)) {
+ /* invalid continuation byte */
+ goto InvalidContinuation3;
}
ch = (ch << 18) + (ch2 << 12) + (ch3 << 6) + ch4 -
((0xF0 << 18) + (0x80 << 12) + (0x80 << 6) + 0x80);
@@ -198,14 +225,14 @@
s += 4;
if (STRINGLIB_MAX_CHAR <= 0xFFFF ||
(STRINGLIB_MAX_CHAR < 0x10FFFF && ch > STRINGLIB_MAX_CHAR))
- goto Overflow;
+ /* Out-of-range */
+ goto Return;
*p++ = ch;
continue;
}
goto InvalidStart;
}
ch = 0;
-Overflow:
Return:
*inptr = s;
*outpos = p - dest;
@@ -213,13 +240,18 @@
InvalidStart:
ch = 1;
goto Return;
-InvalidContinuation:
+InvalidContinuation1:
ch = 2;
goto Return;
+InvalidContinuation2:
+ ch = 3;
+ goto Return;
+InvalidContinuation3:
+ ch = 4;
+ goto Return;
}
#undef ASCII_CHAR_MASK
-#undef IS_CONTINUATION_BYTE
/* UTF-8 encoder specialized for a Unicode kind to avoid the slow
diff -r ec656d79b8ac Objects/unicodeobject.c
--- a/Objects/unicodeobject.c Sat Jun 23 22:55:58 2012 +0200
+++ b/Objects/unicodeobject.c Sun Jun 24 00:20:16 2012 +0300
@@ -4763,9 +4763,7 @@
goto End;
errmsg = "unexpected end of data";
startinpos = s - starts;
- endinpos = startinpos + 1;
- while (endinpos < size && (starts[endinpos] & 0xC0) == 0x80)
- endinpos++;
+ endinpos = end - starts;
break;
case 1:
errmsg = "invalid start byte";
@@ -4773,11 +4771,11 @@
endinpos = startinpos + 1;
break;
case 2:
+ case 3:
+ case 4:
errmsg = "invalid continuation byte";
startinpos = s - starts;
- endinpos = startinpos + 1;
- while (endinpos < size && (starts[endinpos] & 0xC0) == 0x80)
- endinpos++;
+ endinpos = startinpos + ch - 1;
break;
default:
if (unicode_putchar(&unicode, &outpos, ch) < 0)
_______________________________________________
Python-bugs-list mailing list
Unsubscribe:
http://mail.python.org/mailman/options/python-bugs-list/archive%40mail-archive.com
