[pypy-commit] pypy py3k: properly decode *all* AST identifiers

[pjenvey]

Author: Philip Jenvey <pjen...@underboss.org>
Branch: py3k
Changeset: r62323:80061bd0e2b6
Date: 2013-03-12 16:03 -0700
http://bitbucket.org/pypy/pypy/changeset/80061bd0e2b6/

Log:    properly decode *all* AST identifiers

diff --git a/pypy/interpreter/astcompiler/ast.py 
b/pypy/interpreter/astcompiler/ast.py
--- a/pypy/interpreter/astcompiler/ast.py
+++ b/pypy/interpreter/astcompiler/ast.py
@@ -3105,7 +3105,9 @@
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
         raise operationerrfmt(space.w_AttributeError, "'%s' object has no 
attribute '%s'", typename, 'name')
-    return space.wrap(w_self.name)
+    if w_self.name is None:
+        return space.w_None
+    return space.wrap(w_self.name.decode('utf-8'))
 
 def FunctionDef_set_name(space, w_self, w_new_value):
     try:
@@ -3237,7 +3239,9 @@
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
         raise operationerrfmt(space.w_AttributeError, "'%s' object has no 
attribute '%s'", typename, 'name')
-    return space.wrap(w_self.name)
+    if w_self.name is None:
+        return space.w_None
+    return space.wrap(w_self.name.decode('utf-8'))
 
 def ClassDef_set_name(space, w_self, w_new_value):
     try:
@@ -4368,7 +4372,9 @@
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
         raise operationerrfmt(space.w_AttributeError, "'%s' object has no 
attribute '%s'", typename, 'module')
-    return space.wrap(w_self.module)
+    if w_self.module is None:
+        return space.w_None
+    return space.wrap(w_self.module.decode('utf-8'))
 
 def ImportFrom_set_module(space, w_self, w_new_value):
     try:
@@ -5947,7 +5953,9 @@
     if not w_self.initialization_state & 8:
         typename = space.type(w_self).getname(space)
         raise operationerrfmt(space.w_AttributeError, "'%s' object has no 
attribute '%s'", typename, 'attr')
-    return space.wrap(w_self.attr)
+    if w_self.attr is None:
+        return space.w_None
+    return space.wrap(w_self.attr.decode('utf-8'))
 
 def Attribute_set_attr(space, w_self, w_new_value):
     try:
@@ -6190,7 +6198,9 @@
     if not w_self.initialization_state & 4:
         typename = space.type(w_self).getname(space)
         raise operationerrfmt(space.w_AttributeError, "'%s' object has no 
attribute '%s'", typename, 'id')
-    return space.wrap(w_self.id)
+    if w_self.id is None:
+        return space.w_None
+    return space.wrap(w_self.id.decode('utf-8'))
 
 def Name_set_id(space, w_self, w_new_value):
     try:
@@ -7078,7 +7088,9 @@
     if not w_self.initialization_state & 8:
         typename = space.type(w_self).getname(space)
         raise operationerrfmt(space.w_AttributeError, "'%s' object has no 
attribute '%s'", typename, 'name')
-    return space.wrap(w_self.name)
+    if w_self.name is None:
+        return space.w_None
+    return space.wrap(w_self.name.decode('utf-8'))
 
 def ExceptHandler_set_name(space, w_self, w_new_value):
     try:
@@ -7164,7 +7176,9 @@
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
         raise operationerrfmt(space.w_AttributeError, "'%s' object has no 
attribute '%s'", typename, 'vararg')
-    return space.wrap(w_self.vararg)
+    if w_self.vararg is None:
+        return space.w_None
+    return space.wrap(w_self.vararg.decode('utf-8'))
 
 def arguments_set_vararg(space, w_self, w_new_value):
     try:
@@ -7230,7 +7244,9 @@
     if not w_self.initialization_state & 16:
         typename = space.type(w_self).getname(space)
         raise operationerrfmt(space.w_AttributeError, "'%s' object has no 
attribute '%s'", typename, 'kwarg')
-    return space.wrap(w_self.kwarg)
+    if w_self.kwarg is None:
+        return space.w_None
+    return space.wrap(w_self.kwarg.decode('utf-8'))
 
 def arguments_set_kwarg(space, w_self, w_new_value):
     try:
@@ -7348,7 +7364,9 @@
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
         raise operationerrfmt(space.w_AttributeError, "'%s' object has no 
attribute '%s'", typename, 'arg')
-    return space.wrap(w_self.arg)
+    if w_self.arg is None:
+        return space.w_None
+    return space.wrap(w_self.arg.decode('utf-8'))
 
 def arg_set_arg(space, w_self, w_new_value):
     try:
@@ -7419,7 +7437,9 @@
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
         raise operationerrfmt(space.w_AttributeError, "'%s' object has no 
attribute '%s'", typename, 'arg')
-    return space.wrap(w_self.arg)
+    if w_self.arg is None:
+        return space.w_None
+    return space.wrap(w_self.arg.decode('utf-8'))
 
 def keyword_set_arg(space, w_self, w_new_value):
     try:
@@ -7490,7 +7510,9 @@
     if not w_self.initialization_state & 1:
         typename = space.type(w_self).getname(space)
         raise operationerrfmt(space.w_AttributeError, "'%s' object has no 
attribute '%s'", typename, 'name')
-    return space.wrap(w_self.name)
+    if w_self.name is None:
+        return space.w_None
+    return space.wrap(w_self.name.decode('utf-8'))
 
 def alias_set_name(space, w_self, w_new_value):
     try:
@@ -7512,7 +7534,9 @@
     if not w_self.initialization_state & 2:
         typename = space.type(w_self).getname(space)
         raise operationerrfmt(space.w_AttributeError, "'%s' object has no 
attribute '%s'", typename, 'asname')
-    return space.wrap(w_self.asname)
+    if w_self.asname is None:
+        return space.w_None
+    return space.wrap(w_self.asname.decode('utf-8'))
 
 def alias_set_asname(space, w_self, w_new_value):
     try:
diff --git a/pypy/interpreter/astcompiler/tools/asdl_py.py 
b/pypy/interpreter/astcompiler/tools/asdl_py.py
--- a/pypy/interpreter/astcompiler/tools/asdl_py.py
+++ b/pypy/interpreter/astcompiler/tools/asdl_py.py
@@ -438,6 +438,10 @@
             self.emit("return %s_to_class[w_self.%s - 1]()" % config, 1)
         elif field.type.value in ("object", "string"):
             self.emit("return w_self.%s" % (field.name,), 1)
+        elif field.type.value == 'identifier':
+            self.emit("if w_self.%s is None:" % (field.name,), 1)
+            self.emit("return space.w_None", 2)
+            self.emit("return space.wrap(w_self.%s.decode('utf-8'))" % 
(field.name,), 1)
         else:
             self.emit("return space.wrap(w_self.%s)" % (field.name,), 1)
         self.emit("")
diff --git a/pypy/module/_ast/test/test_ast.py 
b/pypy/module/_ast/test/test_ast.py
--- a/pypy/module/_ast/test/test_ast.py
+++ b/pypy/module/_ast/test/test_ast.py
@@ -86,10 +86,29 @@
         name.id = "hi"
         assert name.id == "hi"
 
-    def test_unicode_identifier(self):
+    def test_name_pep3131(self):
         name = self.get_ast("&#26085;&#26412;", "eval").body
+        assert isinstance(name, self.ast.Name)
         assert name.id == "&#26085;&#26412;"
 
+    def test_function_pep3131(self):
+        fn = self.get_ast("def &#181;(&#181;='foo'): pass").body[0]
+        assert isinstance(fn, self.ast.FunctionDef)
+        # &#181; normalized to NFKC
+        expected = '\u03bc'
+        assert fn.name == expected
+        assert fn.args.args[0].arg == expected
+
+    def test_import_pep3131(self):
+        ast = self.ast
+        im = self.get_ast("from package&#181; import mod&#181; as 
&#181;").body[0]
+        assert isinstance(im, ast.ImportFrom)
+        expected = '\u03bc'
+        assert im.module == 'package' + expected
+        alias = im.names[0]
+        assert alias.name == 'mod' + expected
+        assert alias.asname == expected
+
     @py.test.mark.skipif("py.test.config.option.runappdirect")
     def test_object(self):
         ast = self.ast
_______________________________________________
pypy-commit mailing list
pypy-commit@python.org
http://mail.python.org/mailman/listinfo/pypy-commit