Serhiy Storchaka added the comment:
Patch updated to resolve conflict with issue15379. Added tests. Added patches
for 3.2 and 2.7.
----------
Added file: http://bugs.python.org/file27387/decode_charmap_fffe-3.3.patch
Added file: http://bugs.python.org/file27388/decode_charmap_fffe-3.2.patch
Added file: http://bugs.python.org/file27389/decode_charmap_fffe-2.7.patch
_______________________________________
Python tracker <rep...@bugs.python.org>
<http://bugs.python.org/issue14850>
_______________________________________
diff -r 5ddc7b3f2795 Lib/test/test_codecs.py
--- a/Lib/test/test_codecs.py Tue Oct 02 12:54:07 2012 +0200
+++ b/Lib/test/test_codecs.py Tue Oct 02 19:07:20 2012 +0300
@@ -1701,6 +1701,10 @@
codecs.charmap_decode, b"\x00\x01\x02", "strict", "ab"
)
+ self.assertRaises(UnicodeDecodeError,
+ codecs.charmap_decode, b"\x00\x01\x02", "strict", "ab\ufffe"
+ )
+
self.assertEqual(
codecs.charmap_decode(b"\x00\x01\x02", "replace", "ab"),
("ab\ufffd", 3)
@@ -1757,6 +1761,12 @@
{0: 'a', 1: 'b'}
)
+ # Issue #14850
+ self.assertRaises(UnicodeDecodeError,
+ codecs.charmap_decode, b"\x00\x01\x02", "strict",
+ {0: 'a', 1: 'b', 3: '\ufffe'}
+ )
+
self.assertEqual(
codecs.charmap_decode(b"\x00\x01\x02", "replace",
{0: 'a', 1: 'b'}),
@@ -1769,6 +1779,13 @@
("ab\ufffd", 3)
)
+ # Issue #14850
+ self.assertEqual(
+ codecs.charmap_decode(b"\x00\x01\x02", "replace",
+ {0: 'a', 1: 'b', 2: '\ufffe'}),
+ ("ab\ufffd", 3)
+ )
+
self.assertEqual(
codecs.charmap_decode(b"\x00\x01\x02", "ignore",
{0: 'a', 1: 'b'}),
@@ -1781,6 +1798,13 @@
("ab", 3)
)
+ # Issue #14850
+ self.assertEqual(
+ codecs.charmap_decode(b"\x00\x01\x02", "ignore",
+ {0: 'a', 1: 'b', 2: '\ufffe'}),
+ ("ab", 3)
+ )
+
allbytes = bytes(range(256))
self.assertEqual(
codecs.charmap_decode(allbytes, "ignore", {}),
@@ -1821,6 +1845,11 @@
{0: a, 1: b},
)
+ self.assertRaises(UnicodeDecodeError,
+ codecs.charmap_decode, b"\x00\x01\x02", "strict",
+ {0: a, 1: b, 2: 0xFFFE},
+ )
+
self.assertEqual(
codecs.charmap_decode(b"\x00\x01\x02", "replace",
{0: a, 1: b}),
@@ -1828,11 +1857,23 @@
)
self.assertEqual(
+ codecs.charmap_decode(b"\x00\x01\x02", "replace",
+ {0: a, 1: b, 2: 0xFFFE}),
+ ("ab\ufffd", 3)
+ )
+
+ self.assertEqual(
codecs.charmap_decode(b"\x00\x01\x02", "ignore",
{0: a, 1: b}),
("ab", 3)
)
+ self.assertEqual(
+ codecs.charmap_decode(b"\x00\x01\x02", "ignore",
+ {0: a, 1: b, 2: 0xFFFE}),
+ ("ab", 3)
+ )
+
class WithStmtTest(unittest.TestCase):
def test_encodedfile(self):
diff -r 5ddc7b3f2795 Objects/unicodeobject.c
--- a/Objects/unicodeobject.c Tue Oct 02 12:54:07 2012 +0200
+++ b/Objects/unicodeobject.c Tue Oct 02 19:07:20 2012 +0300
@@ -7516,15 +7516,18 @@
if (PyErr_ExceptionMatches(PyExc_LookupError)) {
/* No mapping found means: mapping is undefined. */
PyErr_Clear();
- x = Py_None;
- Py_INCREF(x);
+ goto Undefined;
} else
goto onError;
}
/* Apply mapping */
+ if (x == Py_None)
+ goto Undefined;
if (PyLong_Check(x)) {
long value = PyLong_AS_LONG(x);
+ if (value == 0xFFFE)
+ goto Undefined;
if (value < 0 || value > MAX_UNICODE) {
PyErr_Format(PyExc_TypeError,
"character mapping must be in range(0x%lx)",
@@ -7535,21 +7538,6 @@
if (unicode_putchar(&v, &outpos, value) < 0)
goto onError;
}
- else if (x == Py_None) {
- /* undefined mapping */
- startinpos = s-starts;
- endinpos = startinpos+1;
- if (unicode_decode_call_errorhandler(
- errors, &errorHandler,
- "charmap", "character maps to <undefined>",
- &starts, &e, &startinpos, &endinpos, &exc, &s,
- &v, &outpos)) {
- Py_DECREF(x);
- goto onError;
- }
- Py_DECREF(x);
- continue;
- }
else if (PyUnicode_Check(x)) {
Py_ssize_t targetsize;
@@ -7559,8 +7547,10 @@
if (targetsize == 1) {
/* 1-1 mapping */
- if (unicode_putchar(&v, &outpos,
- PyUnicode_READ_CHAR(x, 0)) < 0)
+ Py_UCS4 value = PyUnicode_READ_CHAR(x, 0);
+ if (value == 0xFFFE)
+ goto Undefined;
+ if (unicode_putchar(&v, &outpos, value) < 0)
goto onError;
}
else if (targetsize > 1) {
@@ -7595,6 +7585,19 @@
}
Py_DECREF(x);
++s;
+ continue;
+Undefined:
+ /* undefined mapping */
+ Py_XDECREF(x);
+ startinpos = s-starts;
+ endinpos = startinpos+1;
+ if (unicode_decode_call_errorhandler(
+ errors, &errorHandler,
+ "charmap", "character maps to <undefined>",
+ &starts, &e, &startinpos, &endinpos, &exc, &s,
+ &v, &outpos)) {
+ goto onError;
+ }
}
}
if (unicode_resize(&v, outpos) < 0)
diff -r d5a4300702c1 Lib/test/test_codecs.py
--- a/Lib/test/test_codecs.py Tue Oct 02 05:35:39 2012 +0300
+++ b/Lib/test/test_codecs.py Tue Oct 02 19:01:39 2012 +0300
@@ -1550,6 +1550,10 @@
codecs.charmap_decode, b"\x00\x01\x02", "strict", "ab"
)
+ self.assertRaises(UnicodeDecodeError,
+ codecs.charmap_decode, b"\x00\x01\x02", "strict", "ab\ufffe"
+ )
+
self.assertEqual(
codecs.charmap_decode(b"\x00\x01\x02", "replace", "ab"),
("ab\ufffd", 3)
@@ -1606,6 +1610,12 @@
{0: 'a', 1: 'b'}
)
+ # Issue #14850
+ self.assertRaises(UnicodeDecodeError,
+ codecs.charmap_decode, b"\x00\x01\x02", "strict",
+ {0: 'a', 1: 'b', 3: '\ufffe'}
+ )
+
self.assertEqual(
codecs.charmap_decode(b"\x00\x01\x02", "replace",
{0: 'a', 1: 'b'}),
@@ -1618,6 +1628,13 @@
("ab\ufffd", 3)
)
+ # Issue #14850
+ self.assertEqual(
+ codecs.charmap_decode(b"\x00\x01\x02", "replace",
+ {0: 'a', 1: 'b', 2: '\ufffe'}),
+ ("ab\ufffd", 3)
+ )
+
self.assertEqual(
codecs.charmap_decode(b"\x00\x01\x02", "ignore",
{0: 'a', 1: 'b'}),
@@ -1630,6 +1647,13 @@
("ab", 3)
)
+ # Issue #14850
+ self.assertEqual(
+ codecs.charmap_decode(b"\x00\x01\x02", "ignore",
+ {0: 'a', 1: 'b', 2: '\ufffe'}),
+ ("ab", 3)
+ )
+
allbytes = bytes(range(256))
self.assertEqual(
codecs.charmap_decode(allbytes, "ignore", {}),
@@ -1664,6 +1688,11 @@
{0: a, 1: b},
)
+ self.assertRaises(UnicodeDecodeError,
+ codecs.charmap_decode, b"\x00\x01\x02", "strict",
+ {0: a, 1: b, 2: 0xFFFE},
+ )
+
self.assertEqual(
codecs.charmap_decode(b"\x00\x01\x02", "replace",
{0: a, 1: b}),
@@ -1671,11 +1700,23 @@
)
self.assertEqual(
+ codecs.charmap_decode(b"\x00\x01\x02", "replace",
+ {0: a, 1: b, 2: 0xFFFE}),
+ ("ab\ufffd", 3)
+ )
+
+ self.assertEqual(
codecs.charmap_decode(b"\x00\x01\x02", "ignore",
{0: a, 1: b}),
("ab", 3)
)
+ self.assertEqual(
+ codecs.charmap_decode(b"\x00\x01\x02", "ignore",
+ {0: a, 1: b, 2: 0xFFFE}),
+ ("ab", 3)
+ )
+
class WithStmtTest(unittest.TestCase):
def test_encodedfile(self):
diff -r d5a4300702c1 Objects/unicodeobject.c
--- a/Objects/unicodeobject.c Tue Oct 02 05:35:39 2012 +0300
+++ b/Objects/unicodeobject.c Tue Oct 02 19:01:39 2012 +0300
@@ -5241,15 +5241,18 @@
if (PyErr_ExceptionMatches(PyExc_LookupError)) {
/* No mapping found means: mapping is undefined. */
PyErr_Clear();
- x = Py_None;
- Py_INCREF(x);
+ goto Undefined;
} else
goto onError;
}
/* Apply mapping */
+ if (x == Py_None)
+ goto Undefined;
if (PyLong_Check(x)) {
long value = PyLong_AS_LONG(x);
+ if (value == 0xFFFE)
+ goto Undefined;
if (value < 0 || value > 0x10FFFF) {
PyErr_SetString(PyExc_TypeError,
"character mapping must be in
range(0x110000)");
@@ -5282,29 +5285,16 @@
#endif
*p++ = (Py_UNICODE)value;
}
- else if (x == Py_None) {
- /* undefined mapping */
- outpos = p-PyUnicode_AS_UNICODE(v);
- startinpos = s-starts;
- endinpos = startinpos+1;
- if (unicode_decode_call_errorhandler(
- errors, &errorHandler,
- "charmap", "character maps to <undefined>",
- &starts, &e, &startinpos, &endinpos, &exc, &s,
- &v, &outpos, &p)) {
- Py_DECREF(x);
- goto onError;
- }
- Py_DECREF(x);
- continue;
- }
else if (PyUnicode_Check(x)) {
Py_ssize_t targetsize = PyUnicode_GET_SIZE(x);
- if (targetsize == 1)
+ if (targetsize == 1) {
/* 1-1 mapping */
- *p++ = *PyUnicode_AS_UNICODE(x);
-
+ Py_UNICODE value = *PyUnicode_AS_UNICODE(x);
+ if (value == 0xFFFE)
+ goto Undefined;
+ *p++ = value;
+ }
else if (targetsize > 1) {
/* 1-n mapping */
if (targetsize > extrachars) {
@@ -5338,6 +5328,20 @@
}
Py_DECREF(x);
++s;
+ continue;
+Undefined:
+ /* undefined mapping */
+ Py_XDECREF(x);
+ outpos = p-PyUnicode_AS_UNICODE(v);
+ startinpos = s-starts;
+ endinpos = startinpos+1;
+ if (unicode_decode_call_errorhandler(
+ errors, &errorHandler,
+ "charmap", "character maps to <undefined>",
+ &starts, &e, &startinpos, &endinpos, &exc, &s,
+ &v, &outpos, &p)) {
+ goto onError;
+ }
}
}
if (p - PyUnicode_AS_UNICODE(v) < PyUnicode_GET_SIZE(v))
diff -r 60c831305e73 Lib/test/test_codecs.py
--- a/Lib/test/test_codecs.py Tue Oct 02 05:34:38 2012 +0300
+++ b/Lib/test/test_codecs.py Tue Oct 02 19:10:22 2012 +0300
@@ -1519,6 +1519,14 @@
(u"abc", 3)
)
+ self.assertRaises(UnicodeDecodeError,
+ codecs.charmap_decode, "\x00\x01\x02", "strict", u"ab"
+ )
+
+ self.assertRaises(UnicodeDecodeError,
+ codecs.charmap_decode, "\x00\x01\x02", "strict", u"ab\ufffe"
+ )
+
self.assertEqual(
codecs.charmap_decode("\x00\x01\x02", "replace", u"ab"),
(u"ab\ufffd", 3)
@@ -1545,6 +1553,139 @@
(u"", len(allbytes))
)
+ def test_decode_with_int2str_map(self):
+ self.assertEqual(
+ codecs.charmap_decode("\x00\x01\x02", "strict",
+ {0: u'a', 1: u'b', 2: u'c'}),
+ (u"abc", 3)
+ )
+
+ self.assertEqual(
+ codecs.charmap_decode("\x00\x01\x02", "strict",
+ {0: u'Aa', 1: u'Bb', 2: u'Cc'}),
+ (u"AaBbCc", 3)
+ )
+
+ self.assertEqual(
+ codecs.charmap_decode("\x00\x01\x02", "strict",
+ {0: u'\U0010FFFF', 1: u'b', 2: u'c'}),
+ (u"\U0010FFFFbc", 3)
+ )
+
+ self.assertEqual(
+ codecs.charmap_decode("\x00\x01\x02", "strict",
+ {0: u'a', 1: u'b', 2: u''}),
+ (u"ab", 3)
+ )
+
+ # Issue #14850
+ self.assertRaises(UnicodeDecodeError,
+ codecs.charmap_decode, "\x00\x01\x02", "strict",
+ {0: u'a', 1: u'b', 3: u'\ufffe'}
+ )
+
+ self.assertEqual(
+ codecs.charmap_decode("\x00\x01\x02", "replace",
+ {0: u'a', 1: u'b'}),
+ (u"ab\ufffd", 3)
+ )
+
+ self.assertEqual(
+ codecs.charmap_decode("\x00\x01\x02", "replace",
+ {0: u'a', 1: u'b', 2: None}),
+ (u"ab\ufffd", 3)
+ )
+
+ # Issue #14850
+ self.assertEqual(
+ codecs.charmap_decode("\x00\x01\x02", "replace",
+ {0: u'a', 1: u'b', 2: u'\ufffe'}),
+ (u"ab\ufffd", 3)
+ )
+
+ self.assertEqual(
+ codecs.charmap_decode("\x00\x01\x02", "ignore",
+ {0: u'a', 1: u'b'}),
+ (u"ab", 3)
+ )
+
+ self.assertEqual(
+ codecs.charmap_decode("\x00\x01\x02", "ignore",
+ {0: u'a', 1: u'b', 2: None}),
+ (u"ab", 3)
+ )
+
+ # Issue #14850
+ self.assertEqual(
+ codecs.charmap_decode("\x00\x01\x02", "ignore",
+ {0: u'a', 1: u'b', 2: u'\ufffe'}),
+ (u"ab", 3)
+ )
+
+ allbytes = "".join(chr(i) for i in xrange(256))
+ self.assertEqual(
+ codecs.charmap_decode(allbytes, "ignore", {}),
+ (u"", len(allbytes))
+ )
+
+ def test_decode_with_int2int_map(self):
+ a = ord(u'a')
+ b = ord(u'b')
+ c = ord(u'c')
+
+ self.assertEqual(
+ codecs.charmap_decode("\x00\x01\x02", "strict",
+ {0: a, 1: b, 2: c}),
+ (u"abc", 3)
+ )
+
+ # Issue #15379
+ #self.assertEqual(
+ # codecs.charmap_decode("\x00\x01\x02", "strict",
+ # {0: 0x10FFFF, 1: b, 2: c}),
+ # (u"\U0010FFFFbc", 3)
+ #)
+
+ self.assertRaises(TypeError,
+ codecs.charmap_decode, "\x00\x01\x02", "strict",
+ {0: 0x110000, 1: b, 2: c}
+ )
+
+ self.assertRaises(UnicodeDecodeError,
+ codecs.charmap_decode, "\x00\x01\x02", "strict",
+ {0: a, 1: b},
+ )
+
+ self.assertRaises(UnicodeDecodeError,
+ codecs.charmap_decode, "\x00\x01\x02", "strict",
+ {0: a, 1: b, 2: 0xFFFE},
+ )
+
+ self.assertEqual(
+ codecs.charmap_decode("\x00\x01\x02", "replace",
+ {0: a, 1: b}),
+ (u"ab\ufffd", 3)
+ )
+
+ self.assertEqual(
+ codecs.charmap_decode("\x00\x01\x02", "replace",
+ {0: a, 1: b, 2: 0xFFFE}),
+ (u"ab\ufffd", 3)
+ )
+
+ self.assertEqual(
+ codecs.charmap_decode("\x00\x01\x02", "ignore",
+ {0: a, 1: b}),
+ (u"ab", 3)
+ )
+
+ self.assertEqual(
+ codecs.charmap_decode("\x00\x01\x02", "ignore",
+ {0: a, 1: b, 2: 0xFFFE}),
+ (u"ab", 3)
+ )
+
+
class WithStmtTest(unittest.TestCase):
def test_encodedfile(self):
f = StringIO.StringIO("\xc3\xbc")
diff -r 60c831305e73 Objects/unicodeobject.c
--- a/Objects/unicodeobject.c Tue Oct 02 05:34:38 2012 +0300
+++ b/Objects/unicodeobject.c Tue Oct 02 19:10:22 2012 +0300
@@ -4118,15 +4118,18 @@
if (PyErr_ExceptionMatches(PyExc_LookupError)) {
/* No mapping found means: mapping is undefined. */
PyErr_Clear();
- x = Py_None;
- Py_INCREF(x);
+ goto Undefined;
} else
goto onError;
}
/* Apply mapping */
+ if (x == Py_None)
+ goto Undefined;
if (PyInt_Check(x)) {
long value = PyInt_AS_LONG(x);
+ if (value == 0xFFFE)
+ goto Undefined;
if (value < 0 || value > 65535) {
PyErr_SetString(PyExc_TypeError,
"character mapping must be in
range(65536)");
@@ -4135,29 +4138,16 @@
}
*p++ = (Py_UNICODE)value;
}
- else if (x == Py_None) {
- /* undefined mapping */
- outpos = p-PyUnicode_AS_UNICODE(v);
- startinpos = s-starts;
- endinpos = startinpos+1;
- if (unicode_decode_call_errorhandler(
- errors, &errorHandler,
- "charmap", "character maps to <undefined>",
- starts, size, &startinpos, &endinpos, &exc, &s,
- &v, &outpos, &p)) {
- Py_DECREF(x);
- goto onError;
- }
- Py_DECREF(x);
- continue;
- }
else if (PyUnicode_Check(x)) {
Py_ssize_t targetsize = PyUnicode_GET_SIZE(x);
- if (targetsize == 1)
+ if (targetsize == 1) {
/* 1-1 mapping */
- *p++ = *PyUnicode_AS_UNICODE(x);
-
+ Py_UNICODE value = *PyUnicode_AS_UNICODE(x);
+ if (value == 0xFFFE)
+ goto Undefined;
+ *p++ = value;
+ }
else if (targetsize > 1) {
/* 1-n mapping */
if (targetsize > extrachars) {
@@ -4191,6 +4181,20 @@
}
Py_DECREF(x);
++s;
+ continue;
+Undefined:
+ /* undefined mapping */
+ Py_XDECREF(x);
+ outpos = p-PyUnicode_AS_UNICODE(v);
+ startinpos = s-starts;
+ endinpos = startinpos+1;
+ if (unicode_decode_call_errorhandler(
+ errors, &errorHandler,
+ "charmap", "character maps to <undefined>",
+ starts, size, &startinpos, &endinpos, &exc, &s,
+ &v, &outpos, &p)) {
+ goto onError;
+ }
}
}
if (p - PyUnicode_AS_UNICODE(v) < PyUnicode_GET_SIZE(v))
_______________________________________________
Python-bugs-list mailing list
Unsubscribe:
http://mail.python.org/mailman/options/python-bugs-list/archive%40mail-archive.com
