[pypy-commit] pypy unicode-utf8-py3: fixes from testing module/_ast

[mattip]

Author: Matti Picus 
Branch: unicode-utf8-py3
Changeset: r94797:599273325eea
Date: 2018-07-01 16:37 -0500
http://bitbucket.org/pypy/pypy/changeset/599273325eea/

Log:fixes from testing module/_ast

diff --git a/pypy/objspace/std/bytesobject.py b/pypy/objspace/std/bytesobject.py
--- a/pypy/objspace/std/bytesobject.py
+++ b/pypy/objspace/std/bytesobject.py
@@ -14,6 +14,8 @@
 from pypy.objspace.std.stringmethods import StringMethods
 from pypy.objspace.std.util import IDTAG_SPECIAL, IDTAG_SHIFT
 from pypy.objspace.std.formatting import mod_format, FORMAT_BYTES
+from pypy.objspace.std.unicodeobject import (encode_object, getdefaultencoding,
+   decode_object)
 
 class W_AbstractBytesObject(W_Root):
 __slots__ = ()
@@ -688,7 +690,6 @@
 raise oefmt(space.w_TypeError,
 "encoding without string argument (got '%T' instead)",
 w_source)
-from pypy.objspace.std.unicodeobject import encode_object
 w_source = encode_object(space, w_source, encoding, errors)
 # and continue with the encoded string
 elif errors is not None:
diff --git a/pypy/objspace/std/unicodeobject.py 
b/pypy/objspace/std/unicodeobject.py
--- a/pypy/objspace/std/unicodeobject.py
+++ b/pypy/objspace/std/unicodeobject.py
@@ -1144,14 +1144,13 @@
 
 
 def encode_object(space, w_object, encoding, errors):
-w_encoder = None
-if encoding is None:
-# Get the encoder functions as a wrapped object.
-# This lookup is cached.
-w_encoder = space.sys.get_w_default_encoder()
 if errors is None or errors == 'strict':
-if ((encoding is None and space.sys.defaultencoding == 'ascii') or
- encoding == 'ascii'):
+if encoding is None or encoding == 'utf-8':
+utf8 = space.utf8_w(w_object)
+if rutf8.has_surrogates(utf8):
+utf8 = rutf8.reencode_utf8_with_surrogates(utf8)
+return space.newbytes(utf8)
+elif encoding == 'ascii':
 s = space.utf8_w(w_object)
 try:
 rutf8.check_ascii(s)
@@ -1161,21 +1160,11 @@
 a.pos, a.pos + 1)
 assert False, "always raises"
 return space.newbytes(s)
-if ((encoding is None and space.sys.defaultencoding == 'utf8') or
- encoding == 'utf-8' or encoding == 'utf8' or encoding == 'UTF-8'):
-utf8 = space.utf8_w(w_object)
-if rutf8.has_surrogates(utf8):
-utf8 = rutf8.reencode_utf8_with_surrogates(utf8)
-return space.newbytes(utf8)
-if w_encoder is None:
-from pypy.module._codecs.interp_codecs import lookup_codec
-w_encoder = space.getitem(lookup_codec(space, encoding), 
space.newint(0))
-if errors is None:
-w_errors = space.newtext('strict')
-else:
-w_errors = space.newtext(errors)
-w_restuple = space.call_function(w_encoder, w_object, w_errors)
-w_retval = space.getitem(w_restuple, space.newint(0))
+
+from pypy.module._codecs.interp_codecs import encode_text
+if encoding is None:
+encoding = space.sys.defaultencoding
+w_retval = encode_text(space, w_object, encoding, errors)
 if not space.isinstance_w(w_retval, space.w_bytes):
 raise oefmt(space.w_TypeError,
 "'%s' encoder returned '%T' instead of 'bytes'; "
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy unicode-utf8-py3: fixes from module/_sre/test

[mattip]

Author: Matti Picus 
Branch: unicode-utf8-py3
Changeset: r94802:1a502b1e961b
Date: 2018-07-01 23:11 -0500
http://bitbucket.org/pypy/pypy/changeset/1a502b1e961b/

Log:fixes from module/_sre/test

diff --git a/pypy/module/_sre/interp_sre.py b/pypy/module/_sre/interp_sre.py
--- a/pypy/module/_sre/interp_sre.py
+++ b/pypy/module/_sre/interp_sre.py
@@ -47,6 +47,8 @@
 s = ctx._utf8[start:end]
 lgt = rutf8.get_utf8_length(s)
 return space.newutf8(s, lgt)
+elif isinstance(ctx, rsre_core.UnicodeMatchContext):
+return space.newtext(ctx._unicodestr[start:end])
 else:
 # unreachable
 raise SystemError
@@ -742,6 +744,8 @@
 elif isinstance(ctx, rsre_utf8.Utf8MatchContext):
 lgt = rutf8.get_utf8_length(ctx._utf8)
 return space.newutf8(ctx._utf8, lgt)
+elif isinstance(ctx, rsre_core.UnicodeMatchContext):
+return space.newtext(ctx._unicodestr)
 else:
 raise SystemError
 
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy unicode-utf8: merged default into branch

[mattip]

Author: Matti Picus 
Branch: unicode-utf8
Changeset: r94801:f32c591c9e7f
Date: 2018-07-01 22:45 -0500
http://bitbucket.org/pypy/pypy/changeset/f32c591c9e7f/

Log:merged default into branch

diff --git a/pypy/doc/sandbox.rst b/pypy/doc/sandbox.rst
--- a/pypy/doc/sandbox.rst
+++ b/pypy/doc/sandbox.rst
@@ -3,6 +3,11 @@
 PyPy's sandboxing features
 ==
 
+.. warning:: This is not actively maintained. You will likely have to fix
+   some issues yourself, or otherwise play around on your own. We provide
+   this documentation for historical reasions, it will not translate or
+   run on the latest PyPy code base.
+
 Introduction
 
 
diff --git a/pypy/interpreter/pyparser/automata.py 
b/pypy/interpreter/pyparser/automata.py
--- a/pypy/interpreter/pyparser/automata.py
+++ b/pypy/interpreter/pyparser/automata.py
@@ -23,6 +23,10 @@
 
 ERROR_STATE = chr(255)
 
+# NB: all non-ascii bytes (>= 128) will be turned into 128
+NON_ASCII = chr(128)
+
+
 class DFA:
 # 
 def __init__(self, states, accepts, start = 0):
@@ -36,7 +40,10 @@
 for key in state:
 if key == DEFAULT:
 continue
-maximum = max(ord(key), maximum)
+ordkey = ord(key)
+if ordkey > 128:
+raise ValueError("DFA does not support matching of 
specific non-ASCII character %r. Use NON_ASCII instead" % key)
+maximum = max(ordkey, maximum)
 self.max_char = maximum + 1
 
 defaults = []
@@ -72,6 +79,8 @@
 i = pos
 for i in range(pos, len(inVec)):
 item = inVec[i]
+if ord(item) > 0x80:
+item = NON_ASCII
 accept = self.accepts[crntState]
 crntState = self._next_state(item, crntState)
 if crntState != ERROR_STATE:
@@ -103,6 +112,8 @@
 i = pos
 for i in range(pos, len(inVec)):
 item = inVec[i]
+if ord(item) > 0x80:
+item = NON_ASCII
 accept = self.accepts[crntState]
 if accept:
 return i
diff --git a/pypy/interpreter/pyparser/test/test_automata.py 
b/pypy/interpreter/pyparser/test/test_automata.py
--- a/pypy/interpreter/pyparser/test/test_automata.py
+++ b/pypy/interpreter/pyparser/test/test_automata.py
@@ -1,4 +1,7 @@
-from pypy.interpreter.pyparser.automata import DFA, NonGreedyDFA, DEFAULT
+# coding: utf-8
+import pytest
+
+from pypy.interpreter.pyparser.automata import DFA, NonGreedyDFA, DEFAULT, 
NON_ASCII
 
 def test_states():
 d = DFA([{"\x00": 1}, {"\x01": 0}], [False, True])
@@ -27,3 +30,18 @@
 d = NonGreedyDFA([{"a": 1}, {DEFAULT: 0}], [False, True])
 assert d.recognize("a,a?ab") == 1
 assert d.recognize("c") == -1
+
+def test_nonascii():
+d = DFA([{"a": 1}, {NON_ASCII: 1}], [False, True])
+input = u"a".encode("utf-8")
+assert d.recognize(input) == len(input)
+assert d.recognize("c") == -1
+assert d.recognize("ü") == -1
+
+d = NonGreedyDFA([{NON_ASCII: 0, "b": 1}, {"b": 0}], [False, True])
+input = u"üü".encode("utf-8")
+assert d.recognize(input) == len(u"üüb".encode("utf-8"))
+assert d.recognize("c") == -1
+
+pytest.raises(ValueError, DFA, [{"\x81": 2}], [True])
+
diff --git a/pypy/interpreter/test/test_function.py 
b/pypy/interpreter/test/test_function.py
--- a/pypy/interpreter/test/test_function.py
+++ b/pypy/interpreter/test/test_function.py
@@ -455,6 +455,8 @@
 assert repr(B().f).startswith(">")
 
+assert repr(type(A.f)) == repr(type(A().f)) == ""
+
 
 def test_method_call(self):
 class C(object):
diff --git a/pypy/interpreter/test/test_raise.py 
b/pypy/interpreter/test/test_raise.py
--- a/pypy/interpreter/test/test_raise.py
+++ b/pypy/interpreter/test/test_raise.py
@@ -280,3 +280,15 @@
 def __new__(cls, *args):
 return object()
 raises(TypeError, "raise MyException")
+
+def test_with_exit_True(self):
+class X:
+def __enter__(self):
+pass
+def __exit__(self, *args):
+return True
+def g():
+with X():
+return 42
+assert False, "unreachable"
+assert g() == 42
diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py
--- a/pypy/interpreter/typedef.py
+++ b/pypy/interpreter/typedef.py
@@ -688,7 +688,7 @@
 Function.typedef.acceptable_as_base_class = False
 
 Method.typedef = TypeDef(
-"method",
+"instancemethod",
 __doc__ = """instancemethod(function, instance, class)
 
 Create an instance method object.""",
diff --git a/pypy/module/_cppyy/converter.py b/pypy/module/_cppyy/converter.py
--- a/pypy/module/_cppyy/converter.py
+++ b/pypy/module/_cppyy/converter.py
@@ -706,7 +706,7 @@
 "no overload found matching %s", self.signature)
 
 
-class

[pypy-commit] pypy unicode-utf8-py3: fix interp_stringio for utf8

[mattip]

Author: Matti Picus 
Branch: unicode-utf8-py3
Changeset: r94796:9613305bf7cb
Date: 2018-07-01 11:12 -0500
http://bitbucket.org/pypy/pypy/changeset/9613305bf7cb/

Log:fix interp_stringio for utf8

diff --git a/pypy/module/_io/interp_stringio.py 
b/pypy/module/_io/interp_stringio.py
--- a/pypy/module/_io/interp_stringio.py
+++ b/pypy/module/_io/interp_stringio.py
@@ -46,7 +46,7 @@
 limit = self._convert_limit(limit)
 start = self.pos
 if start >= len(self.data):
-return u''
+return ''
 end = start + limit
 pos = start
 while pos < end:
@@ -70,7 +70,7 @@
 start = self.pos
 limit = self._convert_limit(limit)
 if start >= len(self.data):
-return u''
+return ''
 end = start + limit
 found = False
 for pos in range(start, end - len(marker) + 1):
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy unicode-utf8-py3: merge py3.5 into branch

[mattip]

Author: Matti Picus 
Branch: unicode-utf8-py3
Changeset: r94800:ebb1b24a3065
Date: 2018-07-01 22:43 -0500
http://bitbucket.org/pypy/pypy/changeset/ebb1b24a3065/

Log:merge py3.5 into branch

diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -1260,9 +1260,11 @@
 
 def WITH_CLEANUP_FINISH(self, oparg, next_instr):
 w_suppress = self.popvalue()
-if self.space.is_true(w_suppress):
-# __exit__() returned True -> Swallow the exception.
-self.settopvalue(self.space.w_None)
+w_unroller = self.peekvalue()
+if isinstance(w_unroller, SApplicationException):
+if self.space.is_true(w_suppress):
+# __exit__() returned True -> Swallow the exception.
+self.settopvalue(self.space.w_None)
 # this is always followed by END_FINALLY
 # in the stack now: [w_unroller-or-w_None..]
 
diff --git a/pypy/interpreter/test/test_coroutine.py 
b/pypy/interpreter/test/test_coroutine.py
--- a/pypy/interpreter/test/test_coroutine.py
+++ b/pypy/interpreter/test/test_coroutine.py
@@ -112,6 +112,27 @@
 assert seen == ['aenter', 'aexit']
 """
 
+def test_async_with_exit_True(self): """
+seen = []
+class X:
+async def __aenter__(self):
+seen.append('aenter')
+async def __aexit__(self, *args):
+seen.append('aexit')
+return True
+async def f(x):
+async with x:
+return 42
+c = f(X())
+try:
+c.send(None)
+except StopIteration as e:
+assert e.value == 42
+else:
+assert False, "should have raised"
+assert seen == ['aenter', 'aexit']
+"""
+
 def test_await(self): """
 class X:
 def __await__(self):
diff --git a/pypy/interpreter/test/test_raise.py 
b/pypy/interpreter/test/test_raise.py
--- a/pypy/interpreter/test/test_raise.py
+++ b/pypy/interpreter/test/test_raise.py
@@ -309,6 +309,17 @@
 return object()
 raises(TypeError, "raise MyException")
 
+def test_with_exit_True(self):
+class X:
+def __enter__(self):
+pass
+def __exit__(self, *args):
+return True
+def g():
+with X():
+return 42
+assert False, "unreachable"
+assert g() == 42
 
 def test_pop_exception_value(self):
 # assert that this code don't crash
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy unicode-utf8-py3: fixes from module/_sre/test

[mattip]

Author: Matti Picus 
Branch: unicode-utf8-py3
Changeset: r94798:ac5381b72782
Date: 2018-07-01 22:41 -0500
http://bitbucket.org/pypy/pypy/changeset/ac5381b72782/

Log:fixes from module/_sre/test

diff --git a/pypy/module/_sre/interp_sre.py b/pypy/module/_sre/interp_sre.py
--- a/pypy/module/_sre/interp_sre.py
+++ b/pypy/module/_sre/interp_sre.py
@@ -163,8 +163,8 @@
 string = None
 buf = None
 space = self.space
-if space.isinstance_w(w_string, space.w_utf8):
-unicodestr = space.utf8_w(w_string).decode()
+if space.isinstance_w(w_string, space.w_unicode):
+unicodestr = space.utf8_w(w_string).decode('utf8')
 length = len(unicodestr)
 elif space.isinstance_w(w_string, space.w_bytes):
 string = space.bytes_w(w_string)
@@ -176,6 +176,7 @@
 return (length, unicodestr, string, buf)
 
 def make_ctx(self, w_string, pos=0, endpos=sys.maxint, flags=0):
+"""Make a StrMatchContext, BufMatchContext or a Utf8MatchContext for
 searching in the given w_string object."""
 space = self.space
 length, unicodestr, string, buf = self.getstring(w_string)
diff --git a/pypy/objspace/std/objspace.py b/pypy/objspace/std/objspace.py
--- a/pypy/objspace/std/objspace.py
+++ b/pypy/objspace/std/objspace.py
@@ -553,7 +553,7 @@
 return w_obj.listview_utf8()
 if type(w_obj) is W_SetObject or type(w_obj) is W_FrozensetObject:
 return w_obj.listview_utf8()
-if (isinstance(w_obj, W_UnicodeObject) and 
self._uni_uses_no_iter(w_obj)
+if (isinstance(w_obj, W_UnicodeObject) and not 
self._uses_unicode_iter(w_obj)
 and w_obj.is_ascii()):
 return w_obj.listview_utf8()
 if isinstance(w_obj, W_ListObject) and self._uses_list_iter(w_obj):
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy py3.6: merge py3.5 into py3.6

[mattip]

Author: Matti Picus 
Branch: py3.6
Changeset: r94799:289d8154d34d
Date: 2018-07-01 22:42 -0500
http://bitbucket.org/pypy/pypy/changeset/289d8154d34d/

Log:merge py3.5 into py3.6

diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -1287,9 +1287,11 @@
 
 def WITH_CLEANUP_FINISH(self, oparg, next_instr):
 w_suppress = self.popvalue()
-if self.space.is_true(w_suppress):
-# __exit__() returned True -> Swallow the exception.
-self.settopvalue(self.space.w_None)
+w_unroller = self.peekvalue()
+if isinstance(w_unroller, SApplicationException):
+if self.space.is_true(w_suppress):
+# __exit__() returned True -> Swallow the exception.
+self.settopvalue(self.space.w_None)
 # this is always followed by END_FINALLY
 # in the stack now: [w_unroller-or-w_None..]
 
diff --git a/pypy/interpreter/test/test_coroutine.py 
b/pypy/interpreter/test/test_coroutine.py
--- a/pypy/interpreter/test/test_coroutine.py
+++ b/pypy/interpreter/test/test_coroutine.py
@@ -130,6 +130,27 @@
 assert seen == ['aenter', 'aexit']
 """
 
+def test_async_with_exit_True(self): """
+seen = []
+class X:
+async def __aenter__(self):
+seen.append('aenter')
+async def __aexit__(self, *args):
+seen.append('aexit')
+return True
+async def f(x):
+async with x:
+return 42
+c = f(X())
+try:
+c.send(None)
+except StopIteration as e:
+assert e.value == 42
+else:
+assert False, "should have raised"
+assert seen == ['aenter', 'aexit']
+"""
+
 def test_await(self): """
 class X:
 def __await__(self):
diff --git a/pypy/interpreter/test/test_raise.py 
b/pypy/interpreter/test/test_raise.py
--- a/pypy/interpreter/test/test_raise.py
+++ b/pypy/interpreter/test/test_raise.py
@@ -309,6 +309,17 @@
 return object()
 raises(TypeError, "raise MyException")
 
+def test_with_exit_True(self):
+class X:
+def __enter__(self):
+pass
+def __exit__(self, *args):
+return True
+def g():
+with X():
+return 42
+assert False, "unreachable"
+assert g() == 42
 
 def test_pop_exception_value(self):
 # assert that this code don't crash
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy py3.5: hg merge default

[arigo]

Author: Armin Rigo 
Branch: py3.5
Changeset: r94794:bd8889a5d67f
Date: 2018-07-01 22:40 +0200
http://bitbucket.org/pypy/pypy/changeset/bd8889a5d67f/

Log:hg merge default

diff --git a/pypy/interpreter/test/test_raise.py 
b/pypy/interpreter/test/test_raise.py
--- a/pypy/interpreter/test/test_raise.py
+++ b/pypy/interpreter/test/test_raise.py
@@ -309,6 +309,17 @@
 return object()
 raises(TypeError, "raise MyException")
 
+def test_with_exit_True(self):
+class X:
+def __enter__(self):
+pass
+def __exit__(self, *args):
+return True
+def g():
+with X():
+return 42
+assert False, "unreachable"
+assert g() == 42
 
 def test_pop_exception_value(self):
 # assert that this code don't crash
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy py3.5: Add another test for 'async with', and fix for 'with' or 'async with':

[arigo]

Author: Armin Rigo 
Branch: py3.5
Changeset: r94795:0a4016e8a6bc
Date: 2018-07-01 22:50 +0200
http://bitbucket.org/pypy/pypy/changeset/0a4016e8a6bc/

Log:Add another test for 'async with', and fix for 'with' or 'async
with': if they call '__exit__' or '__aexit__' and this returns True,
we must ignore that if we're processing a return/break/continue
instead of a real exception.

diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py
--- a/pypy/interpreter/pyopcode.py
+++ b/pypy/interpreter/pyopcode.py
@@ -1260,9 +1260,11 @@
 
 def WITH_CLEANUP_FINISH(self, oparg, next_instr):
 w_suppress = self.popvalue()
-if self.space.is_true(w_suppress):
-# __exit__() returned True -> Swallow the exception.
-self.settopvalue(self.space.w_None)
+w_unroller = self.peekvalue()
+if isinstance(w_unroller, SApplicationException):
+if self.space.is_true(w_suppress):
+# __exit__() returned True -> Swallow the exception.
+self.settopvalue(self.space.w_None)
 # this is always followed by END_FINALLY
 # in the stack now: [w_unroller-or-w_None..]
 
diff --git a/pypy/interpreter/test/test_coroutine.py 
b/pypy/interpreter/test/test_coroutine.py
--- a/pypy/interpreter/test/test_coroutine.py
+++ b/pypy/interpreter/test/test_coroutine.py
@@ -112,6 +112,27 @@
 assert seen == ['aenter', 'aexit']
 """
 
+def test_async_with_exit_True(self): """
+seen = []
+class X:
+async def __aenter__(self):
+seen.append('aenter')
+async def __aexit__(self, *args):
+seen.append('aexit')
+return True
+async def f(x):
+async with x:
+return 42
+c = f(X())
+try:
+c.send(None)
+except StopIteration as e:
+assert e.value == 42
+else:
+assert False, "should have raised"
+assert seen == ['aenter', 'aexit']
+"""
+
 def test_await(self): """
 class X:
 def __await__(self):
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit

[pypy-commit] pypy default: add test that passes on pypy2 but not pypy3

[arigo]

Author: Armin Rigo 
Branch: 
Changeset: r94793:55061f499bd5
Date: 2018-07-01 22:39 +0200
http://bitbucket.org/pypy/pypy/changeset/55061f499bd5/

Log:add test that passes on pypy2 but not pypy3

diff --git a/pypy/interpreter/test/test_raise.py 
b/pypy/interpreter/test/test_raise.py
--- a/pypy/interpreter/test/test_raise.py
+++ b/pypy/interpreter/test/test_raise.py
@@ -280,3 +280,15 @@
 def __new__(cls, *args):
 return object()
 raises(TypeError, "raise MyException")
+
+def test_with_exit_True(self):
+class X:
+def __enter__(self):
+pass
+def __exit__(self, *args):
+return True
+def g():
+with X():
+return 42
+assert False, "unreachable"
+assert g() == 42
___
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit