Author: Philip Jenvey <pjen...@underboss.org>
Branch: py3k
Changeset: r59270:b85a52c00034
Date: 2012-12-02 14:45 -0800
http://bitbucket.org/pypy/pypy/changeset/b85a52c00034/

Log:    merge default

diff --git a/pypy/module/_codecs/interp_codecs.py 
b/pypy/module/_codecs/interp_codecs.py
--- a/pypy/module/_codecs/interp_codecs.py
+++ b/pypy/module/_codecs/interp_codecs.py
@@ -9,15 +9,19 @@
         self.codec_search_cache = {}
         self.codec_error_registry = {}
         self.codec_need_encodings = True
-        self.decode_error_handler = self.make_errorhandler(space, True)
-        self.encode_error_handler = self.make_errorhandler(space, False)
+        self.decode_error_handler = self.make_decode_errorhandler(space)
+        self.encode_error_handler = self.make_encode_errorhandler(space)
 
         self.unicodedata_handler = None
 
-    def make_errorhandler(self, space, decode):
-        def unicode_call_errorhandler(errors,  encoding, reason, input,
-                                      startpos, endpos):
+    def _make_errorhandler(self, space, decode):
+        def call_errorhandler(errors, encoding, reason, input, startpos,
+                              endpos):
+            """Generic wrapper for calling into error handlers.
 
+            Returns (unicode_or_none, str_or_none, newpos) as error
+            handlers may return unicode or on Python 3, bytes.
+            """
             w_errorhandler = lookup_error(space, errors)
             if decode:
                 w_cls = space.w_UnicodeDecodeError
@@ -56,16 +60,21 @@
                 raise operationerrfmt(
                     space.w_IndexError,
                     "position %d from error handler out of bounds", newpos)
-            if decode:
-                replace = space.unicode_w(w_replace)
-                return replace, newpos
-            else:
-                from pypy.objspace.std.unicodetype import encode_object
-                if space.isinstance_w(w_replace, space.w_unicode):
-                    w_replace = encode_object(space, w_replace, encoding, None)
-                replace = space.bytes_w(w_replace)
-                return replace, newpos
-        return unicode_call_errorhandler
+            replace = space.unicode_w(w_replace)
+            return replace, newpos
+        return call_errorhandler
+
+    def make_decode_errorhandler(self, space):
+        return self._make_errorhandler(space, True)
+
+    def make_encode_errorhandler(self, space):
+        errorhandler = self._make_errorhandler(space, False)
+        def encode_call_errorhandler(errors, encoding, reason, input, startpos,
+                                     endpos):
+            replace, newpos = errorhandler(errors, encoding, reason, input,
+                                           startpos, endpos)
+            return replace, None, newpos
+        return encode_call_errorhandler
 
     def get_unicodedata_handler(self, space):
         if self.unicodedata_handler:
diff --git a/pypy/module/_codecs/test/test_codecs.py 
b/pypy/module/_codecs/test/test_codecs.py
--- a/pypy/module/_codecs/test/test_codecs.py
+++ b/pypy/module/_codecs/test/test_codecs.py
@@ -2,7 +2,7 @@
 
 
 class AppTestCodecs:
-    spaceconfig = dict(usemodules=('unicodedata', 'struct'))
+    spaceconfig = dict(usemodules=('binascii', 'struct', 'unicodedata'))
 
     def test_register_noncallable(self):
         import _codecs
diff --git a/pypy/module/_multibytecodec/c_codecs.py 
b/pypy/module/_multibytecodec/c_codecs.py
--- a/pypy/module/_multibytecodec/c_codecs.py
+++ b/pypy/module/_multibytecodec/c_codecs.py
@@ -279,8 +279,14 @@
             replace = "?"
     else:
         assert errorcb
-        replace, end = errorcb(errors, namecb, reason,
-                               unicodedata, start, end)
+        retu, rets, end = errorcb(errors, namecb, reason,
+                                  unicodedata, start, end)
+        if rets is not None:
+            # py3k only
+            replace = rets
+        else:
+            codec = pypy_cjk_enc_getcodec(encodebuf)
+            replace = encode(codec, retu, "strict", errorcb, namecb)
     inbuf = rffi.get_nonmovingbuffer(replace)
     try:
         r = pypy_cjk_enc_replace_on_error(encodebuf, inbuf, len(replace), end)
diff --git a/pypy/module/_multibytecodec/test/test_app_codecs.py 
b/pypy/module/_multibytecodec/test/test_app_codecs.py
--- a/pypy/module/_multibytecodec/test/test_app_codecs.py
+++ b/pypy/module/_multibytecodec/test/test_app_codecs.py
@@ -102,3 +102,11 @@
         repl = "\u2014"
         s = "\uDDA1".encode("gbk", "test.multi_bad_handler")
         assert s == b'\xA1\xAA'
+
+    def test_encode_custom_error_handler_type(self):
+        import codecs
+        import sys
+        codecs.register_error("test.test_encode_custom_error_handler_type",
+                              lambda e: ('\xc3', e.end))
+        raises(TypeError, u"\uDDA1".encode, "gbk",
+               "test.test_encode_custom_error_handler_type")
diff --git a/pypy/module/_multibytecodec/test/test_c_codecs.py 
b/pypy/module/_multibytecodec/test/test_c_codecs.py
--- a/pypy/module/_multibytecodec/test/test_c_codecs.py
+++ b/pypy/module/_multibytecodec/test/test_c_codecs.py
@@ -122,3 +122,10 @@
     c = getcodec('iso2022_jp')
     s = encode(c, u'\u83ca\u5730\u6642\u592b')
     assert s == '\x1b$B5FCO;~IW\x1b(B' and type(s) is str
+
+def test_encode_custom_error_handler_bytes():
+    c = getcodec("hz")
+    def errorhandler(errors, enc, msg, t, startingpos, endingpos):
+        return None, '\xc3', endingpos
+    s = encode(c, u'abc\u1234def', 'foo', errorhandler)
+    assert '\xc3' in s
diff --git a/pypy/rlib/runicode.py b/pypy/rlib/runicode.py
--- a/pypy/rlib/runicode.py
+++ b/pypy/rlib/runicode.py
@@ -57,9 +57,9 @@
 def default_unicode_error_encode(errors, encoding, msg, u,
                                  startingpos, endingpos):
     if errors == 'replace':
-        return u'?', endingpos
+        return u'?', None, endingpos
     if errors == 'ignore':
-        return u'', endingpos
+        return u'', None, endingpos
     raise UnicodeEncodeError(encoding, u, startingpos, endingpos, msg)
 
 # ____________________________________________________________
@@ -300,10 +300,20 @@
                             _encodeUCS4(result, ch3)
                             continue
                     if not allow_surrogates:
-                        r, pos = errorhandler(errors, 'utf-8',
-                                              'surrogates not allowed',
-                                              s, pos-1, pos)
-                        result.append(r)
+                        ru, rs, pos = errorhandler(errors, 'utf-8',
+                                                   'surrogates not allowed',
+                                                   s, pos-1, pos)
+                        if rs is not None:
+                            # py3k only
+                            result.append(rs)
+                            continue
+                        for ch in ru:
+                            if ord(ch) < 0x80:
+                                result.append(chr(ord(ch)))
+                            else:
+                                errorhandler('strict', 'utf-8',
+                                             'surrogates not allowed',
+                                             s, pos-1, pos)
                         continue
                     # else: Fall through and handles isolated high surrogates
                 result.append((chr((0xe0 | (ch >> 12)))))
@@ -970,9 +980,13 @@
             collend = pos+1
             while collend < len(p) and ord(p[collend]) >= limit:
                 collend += 1
-            r, pos = errorhandler(errors, encoding, reason, p,
-                                  collstart, collend)
-            for ch in r:
+            ru, rs, pos = errorhandler(errors, encoding, reason, p,
+                                       collstart, collend)
+            if rs is not None:
+                # py3k only
+                result.append(rs)
+                continue
+            for ch in ru:
                 if ord(ch) < limit:
                     result.append(chr(ord(ch)))
                 else:
@@ -1042,10 +1056,14 @@
 
         c = mapping.get(ch, '')
         if len(c) == 0:
-            res, pos = errorhandler(errors, "charmap",
-                                    "character maps to <undefined>",
-                                    s, pos, pos + 1)
-            for ch2 in res:
+            ru, rs, pos = errorhandler(errors, "charmap",
+                                       "character maps to <undefined>",
+                                       s, pos, pos + 1)
+            if rs is not None:
+                # py3k only
+                result.append(rs)
+                continue
+            for ch2 in ru:
                 c2 = mapping.get(ch2, '')
                 if len(c2) == 0:
                     errorhandler(
@@ -1644,9 +1662,12 @@
                 pass
             collend += 1
         msg = "invalid decimal Unicode string"
-        r, pos = errorhandler(errors, 'decimal',
-                              msg, s, collstart, collend)
-        for char in r:
+        ru, rs, pos = errorhandler(errors, 'decimal',
+                                   msg, s, collstart, collend)
+        if rs is not None:
+            # py3k only
+            errorhandler('strict', 'decimal', msg, s, collstart, collend)
+        for char in ru:
             ch = ord(char)
             if unicodedb.isspace(ch):
                 result.append(' ')
diff --git a/pypy/rlib/test/test_runicode.py b/pypy/rlib/test/test_runicode.py
--- a/pypy/rlib/test/test_runicode.py
+++ b/pypy/rlib/test/test_runicode.py
@@ -66,12 +66,19 @@
             assert t is s
             assert start == startingpos
             assert stop == endingpos
-            return "42424242", stop
+            return u"42424242", None, stop
         encoder = self.getencoder(encoding)
         result = encoder(s, len(s), "foo!", errorhandler)
         assert called[0]
         assert "42424242" in result
 
+        # ensure bytes results passthru
+        def errorhandler_bytes(errors, enc, msg, t, startingpos,
+                               endingpos):
+            return None, '\xc3', endingpos
+        result = encoder(s, len(s), "foo!", errorhandler_bytes)
+        assert '\xc3' in result
+
     def checkdecodeerror(self, s, encoding, start, stop,
                          addstuff=True, msg=None):
         called = [0]
_______________________________________________
pypy-commit mailing list
pypy-commit@python.org
http://mail.python.org/mailman/listinfo/pypy-commit

Reply via email to