[pypy-commit] pypy py3k: improve pep3120 support

[pjenvey]

Author: Philip Jenvey <pjen...@underboss.org>
Branch: py3k
Changeset: r48979:75461738f371
Date: 2011-11-08 17:48 -0800
http://bitbucket.org/pypy/pypy/changeset/75461738f371/

Log:    improve pep3120 support

diff --git a/pypy/interpreter/astcompiler/test/test_astbuilder.py 
b/pypy/interpreter/astcompiler/test/test_astbuilder.py
--- a/pypy/interpreter/astcompiler/test/test_astbuilder.py
+++ b/pypy/interpreter/astcompiler/test/test_astbuilder.py
@@ -1039,6 +1039,17 @@
         assert isinstance(s, ast.Str)
         assert space.eq_w(s.s, space.wrap(sentence))
 
+    def test_string_pep3120(self):
+        space = self.space
+        japan = u'&#26085;&#26412;'
+        source = u"foo = '%s'" % japan
+        info = pyparse.CompileInfo("<test>", "exec")
+        tree = self.parser.parse_source(source.encode("utf-8"), info)
+        assert info.encoding == "utf-8"
+        s = ast_from_node(space, tree, info).body[0].value
+        assert isinstance(s, ast.Str)
+        assert space.eq_w(s.s, space.wrap(japan))
+
     def test_number(self):
         def get_num(s):
             node = self.get_first_expr(s)
diff --git a/pypy/interpreter/pyparser/pyparse.py 
b/pypy/interpreter/pyparser/pyparse.py
--- a/pypy/interpreter/pyparser/pyparse.py
+++ b/pypy/interpreter/pyparser/pyparse.py
@@ -5,8 +5,6 @@
 
 
 def recode_to_utf8(space, bytes, encoding=None):
-    if encoding is None:
-        encoding = 'utf-8'
     if encoding == 'utf-8':
         return bytes
     w_text = space.call_method(space.wrapbytes(bytes), "decode",
@@ -121,6 +119,8 @@
             textsrc = bytessrc
         else:
             enc = _normalize_encoding(_check_for_encoding(bytessrc))
+            if enc is None:
+                enc = 'utf-8'
             try:
                 textsrc = recode_to_utf8(self.space, bytessrc, enc)
             except OperationError, e:
diff --git a/pypy/interpreter/pyparser/test/test_pyparse.py 
b/pypy/interpreter/pyparser/test/test_pyparse.py
--- a/pypy/interpreter/pyparser/test/test_pyparse.py
+++ b/pypy/interpreter/pyparser/test/test_pyparse.py
@@ -64,6 +64,11 @@
         assert exc.msg == ("'ascii' codec can't decode byte 0xc3 "
                            "in position 16: ordinal not in range(128)")
 
+    def test_encoding_pep3120(self):
+        info = pyparse.CompileInfo("<test>", "exec")
+        tree = self.parse("""foo = '&#26085;&#26412;'""", info=info)
+        assert info.encoding == 'utf-8'
+
     def test_syntax_error(self):
         parse = self.parse
         exc = py.test.raises(SyntaxError, parse, "name another for").value
_______________________________________________
pypy-commit mailing list
pypy-commit@python.org
http://mail.python.org/mailman/listinfo/pypy-commit