Author: Matti Picus <[email protected]>
Branch: winconsoleio
Changeset: r97495:905c730547a7
Date: 2019-09-16 17:20 +0300
http://bitbucket.org/pypy/pypy/changeset/905c730547a7/
Log: merge py3.6 into branch
diff too long, truncating to 2000 out of 5141 lines
diff --git a/lib-python/3/stat.py b/lib-python/3/stat.py
--- a/lib-python/3/stat.py
+++ b/lib-python/3/stat.py
@@ -40,6 +40,10 @@
S_IFIFO = 0o010000 # fifo (named pipe)
S_IFLNK = 0o120000 # symbolic link
S_IFSOCK = 0o140000 # socket file
+# Fallbacks for uncommon platform-specific constants
+S_IFDOOR = 0
+S_IFPORT = 0
+S_IFWHT = 0
# Functions to test for each file type
@@ -71,6 +75,18 @@
"""Return True if mode is from a socket."""
return S_IFMT(mode) == S_IFSOCK
+def S_ISDOOR(mode):
+ """Return True if mode is from a door."""
+ return False
+
+def S_ISPORT(mode):
+ """Return True if mode is from an event port."""
+ return False
+
+def S_ISWHT(mode):
+ """Return True if mode is from a whiteout."""
+ return False
+
# Names for permission bits
S_ISUID = 0o4000 # set UID bit
diff --git a/lib-python/3/test/test_concurrent_futures.py
b/lib-python/3/test/test_concurrent_futures.py
--- a/lib-python/3/test/test_concurrent_futures.py
+++ b/lib-python/3/test/test_concurrent_futures.py
@@ -428,6 +428,7 @@
futures_list.remove(future)
wr = weakref.ref(future)
del future
+ test.support.gc_collect()
self.assertIsNone(wr())
futures_list[0].set_result("test")
@@ -435,6 +436,7 @@
futures_list.remove(future)
wr = weakref.ref(future)
del future
+ test.support.gc_collect()
self.assertIsNone(wr())
if futures_list:
futures_list[0].set_result("test")
@@ -533,6 +535,7 @@
for obj in self.executor.map(make_dummy_object, range(10)):
wr = weakref.ref(obj)
del obj
+ test.support.gc_collect()
self.assertIsNone(wr())
diff --git a/lib-python/3/test/test_readline.py
b/lib-python/3/test/test_readline.py
--- a/lib-python/3/test/test_readline.py
+++ b/lib-python/3/test/test_readline.py
@@ -227,7 +227,7 @@
# See https://cnswww.cns.cwru.edu/php/chet/readline/CHANGES
# - editline: history size is broken on OS X 10.11.6.
# Newer versions were not tested yet.
- @unittest.skipIf(readline._READLINE_VERSION < 0x600,
+ @unittest.skipIf(getattr(readline, "_READLINE_VERSION", 0x601) < 0x600,
"this readline version does not support history-size")
@unittest.skipIf(is_editline,
"editline history size configuration is broken")
diff --git a/lib-python/3/test/test_regrtest.py
b/lib-python/3/test/test_regrtest.py
--- a/lib-python/3/test/test_regrtest.py
+++ b/lib-python/3/test/test_regrtest.py
@@ -260,7 +260,6 @@
self.checkError([opt, '0', '-l'], "don't go together")
self.checkError([opt, '0', '-T'], "don't go together")
self.checkError([opt, '0', '-l'], "don't go together")
- self.checkError([opt, '0', '-M', '4G'], "don't go together")
def test_coverage(self):
for opt in '-T', '--coverage':
diff --git a/lib_pypy/_curses_build.py b/lib_pypy/_curses_build.py
--- a/lib_pypy/_curses_build.py
+++ b/lib_pypy/_curses_build.py
@@ -34,6 +34,13 @@
#define NCURSES_OPAQUE 0
#endif
+
+/* ncurses 6 change behaviour and makes all pointers opaque,
+ lets define backward compatibility. It doesn't harm
+ previous versions */
+
+#define NCURSES_INTERNALS 1
+#define NCURSES_REENTRANT 0
#include <ncurses.h>
#include <panel.h>
#include <term.h>
diff --git a/lib_pypy/_dbm.py b/lib_pypy/_dbm.py
--- a/lib_pypy/_dbm.py
+++ b/lib_pypy/_dbm.py
@@ -149,7 +149,7 @@
lib = CDLL("/usr/lib/libdbm.dylib") # OS X
_platform = 'osx'
-library = "GNU gdbm"
+library = "Berkeley DB"
funcs = {}
_init_func('open', (c_char_p, c_int, c_int), restype=c_void_p)
diff --git a/lib_pypy/_pypy_testcapi.py b/lib_pypy/_pypy_testcapi.py
--- a/lib_pypy/_pypy_testcapi.py
+++ b/lib_pypy/_pypy_testcapi.py
@@ -61,6 +61,8 @@
assert output_dir is not None
from distutils.ccompiler import new_compiler
+ from distutils import log
+ log.set_verbosity(3)
compiler = new_compiler()
compiler.output_dir = output_dir
@@ -72,7 +74,8 @@
ccflags = ['-fPIC', '-Wimplicit-function-declaration']
res = compiler.compile([os.path.join(thisdir, csource)],
include_dirs=[include_dir],
- extra_preargs=ccflags)
+ extra_preargs=ccflags,
+ )
object_filename = res[0]
# set link options
diff --git a/lib_pypy/_stat.py b/lib_pypy/_stat.py
deleted file mode 100644
--- a/lib_pypy/_stat.py
+++ /dev/null
@@ -1,6 +0,0 @@
-# Assume not Solaris
-
-S_IFDOOR = 0
-
-def S_ISDOOR(mode):
- return False
diff --git a/lib_pypy/_testcapimodule.c b/lib_pypy/_testcapimodule.c
--- a/lib_pypy/_testcapimodule.c
+++ b/lib_pypy/_testcapimodule.c
@@ -3009,6 +3009,8 @@
return PyLong_FromLong(r);
}
+#endif /* PYPY_VERSION */
+
static int
check_time_rounding(int round)
{
@@ -3069,6 +3071,8 @@
return Py_BuildValue("Nl", _PyLong_FromTime_t(sec), nsec);
}
+#ifndef PYPY_VERSION
+
static void
slot_tp_del(PyObject *self)
{
@@ -3902,8 +3906,6 @@
Py_RETURN_NONE;
}
-#ifndef PYPY_VERSION
-
static PyObject *
test_pytime_fromseconds(PyObject *self, PyObject *args)
{
@@ -4022,6 +4024,8 @@
return _PyTime_AsNanosecondsObject(ms);
}
+#ifndef PYPY_VERSION
+
static PyObject*
get_recursion_depth(PyObject *self, PyObject *args)
{
@@ -4445,9 +4449,11 @@
{"crash_no_current_thread", (PyCFunction)crash_no_current_thread,
METH_NOARGS},
#ifndef PYPY_VERSION
{"run_in_subinterp", run_in_subinterp, METH_VARARGS},
+#endif
{"pytime_object_to_time_t", test_pytime_object_to_time_t, METH_VARARGS},
{"pytime_object_to_timeval", test_pytime_object_to_timeval, METH_VARARGS},
{"pytime_object_to_timespec", test_pytime_object_to_timespec,
METH_VARARGS},
+#ifndef PYPY_VERSION
{"with_tp_del", with_tp_del, METH_VARARGS},
#endif
{"create_cfunction", create_cfunction, METH_NOARGS},
@@ -4515,7 +4521,6 @@
return_null_without_error, METH_NOARGS},
{"return_result_with_error",
return_result_with_error, METH_NOARGS},
-#ifndef PYPY_VERSION
{"PyTime_FromSeconds", test_pytime_fromseconds, METH_VARARGS},
{"PyTime_FromSecondsObject", test_pytime_fromsecondsobject, METH_VARARGS},
{"PyTime_AsSecondsDouble", test_pytime_assecondsdouble, METH_VARARGS},
@@ -4525,6 +4530,7 @@
#endif
{"PyTime_AsMilliseconds", test_PyTime_AsMilliseconds, METH_VARARGS},
{"PyTime_AsMicroseconds", test_PyTime_AsMicroseconds, METH_VARARGS},
+#ifndef PYPY_VERSION
{"get_recursion_depth", get_recursion_depth, METH_NOARGS},
{"pymem_buffer_overflow", pymem_buffer_overflow, METH_NOARGS},
{"pymem_api_misuse", pymem_api_misuse, METH_NOARGS},
@@ -4538,10 +4544,10 @@
{"pyobject_fastcalldict", test_pyobject_fastcalldict, METH_VARARGS},
{"pyobject_fastcallkeywords", test_pyobject_fastcallkeywords,
METH_VARARGS},
{"raise_SIGINT_then_send_None", raise_SIGINT_then_send_None, METH_VARARGS},
+#endif
#ifdef W_STOPCODE
{"W_STOPCODE", py_w_stopcode, METH_VARARGS},
#endif
-#endif /* PYPY_VERSION */
{NULL, NULL} /* sentinel */
};
diff --git a/lib_pypy/_tkinter/tklib_build.py b/lib_pypy/_tkinter/tklib_build.py
--- a/lib_pypy/_tkinter/tklib_build.py
+++ b/lib_pypy/_tkinter/tklib_build.py
@@ -36,8 +36,11 @@
for _ver in ['8.6', '8.5', '']:
incdirs = []
linklibs = ['tcl' + _ver, 'tk' + _ver]
- if os.path.isfile(''.join(['/usr/lib/lib', linklibs[1], '.so'])):
- found = True
+ for lib in ['/usr/lib/lib', '/usr/lib64/lib']:
+ if os.path.isfile(''.join([lib, linklibs[1], '.so'])):
+ found = True
+ break
+ if found:
break
if not found:
sys.stderr.write("*** TCL libraries not found! Falling back...\n")
diff --git a/pypy/config/test/test_pypyoption.py
b/pypy/config/test/test_pypyoption.py
--- a/pypy/config/test/test_pypyoption.py
+++ b/pypy/config/test/test_pypyoption.py
@@ -8,14 +8,13 @@
def test_required():
conf = get_pypy_config()
assert not conf.translating
-
assert conf.objspace.usemodules.gc
def test_conflicting_gcrootfinder():
conf = get_pypy_config()
conf.translation.gc = "boehm"
- py.test.raises(ConfigError, "conf.translation.gcrootfinder = 'asmgcc'")
-
+ with py.test.raises(ConfigError):
+ conf.translation.gcrootfinder = 'asmgcc'
def test_frameworkgc():
for name in ["minimark", "semispace"]:
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -70,3 +70,15 @@
.. branch: cryptograhpt-2.7
Update vendored cryptography used for _ssl to 2.7
+
+.. branch: compile_ncurses_tcl_tk_suse_latest
+
+Check for headers and runtime libraries in more locations to support other
linuxes
+
+.. branch: openssl-for-macos
+
+Update _ssl on macos to statically link to openssl-1.1.1c
+
+.. branch: more-cpyext
+
+Add more datetime C functions and definitions
diff --git a/pypy/interpreter/astcompiler/ast.py
b/pypy/interpreter/astcompiler/ast.py
--- a/pypy/interpreter/astcompiler/ast.py
+++ b/pypy/interpreter/astcompiler/ast.py
@@ -2740,9 +2740,9 @@
w_metavar = get_field(space, w_node, 'metavar', False)
w_lineno = get_field(space, w_node, 'lineno', False)
w_col_offset = get_field(space, w_node, 'col_offset', False)
- _metavar = space.int_w(w_metavar)
- _lineno = space.int_w(w_lineno)
- _col_offset = space.int_w(w_col_offset)
+ _metavar = obj_to_int(space, w_metavar, False)
+ _lineno = obj_to_int(space, w_lineno, False)
+ _col_offset = obj_to_int(space, w_col_offset, False)
return RevDBMetaVar(_metavar, _lineno, _col_offset)
State.ast_type('RevDBMetaVar', 'expr', ['metavar'])
@@ -4291,19 +4291,27 @@
class GenericASTVisitor(ASTVisitor):
+ def visited(self, node):
+ pass # base implementation
+
def visit_Module(self, node):
+ self.visited(node)
self.visit_sequence(node.body)
def visit_Interactive(self, node):
+ self.visited(node)
self.visit_sequence(node.body)
def visit_Expression(self, node):
+ self.visited(node)
node.body.walkabout(self)
def visit_Suite(self, node):
+ self.visited(node)
self.visit_sequence(node.body)
def visit_FunctionDef(self, node):
+ self.visited(node)
node.args.walkabout(self)
self.visit_sequence(node.body)
self.visit_sequence(node.decorator_list)
@@ -4311,6 +4319,7 @@
node.returns.walkabout(self)
def visit_AsyncFunctionDef(self, node):
+ self.visited(node)
node.args.walkabout(self)
self.visit_sequence(node.body)
self.visit_sequence(node.decorator_list)
@@ -4318,214 +4327,269 @@
node.returns.walkabout(self)
def visit_ClassDef(self, node):
+ self.visited(node)
self.visit_sequence(node.bases)
self.visit_sequence(node.keywords)
self.visit_sequence(node.body)
self.visit_sequence(node.decorator_list)
def visit_Return(self, node):
+ self.visited(node)
if node.value:
node.value.walkabout(self)
def visit_Delete(self, node):
+ self.visited(node)
self.visit_sequence(node.targets)
def visit_Assign(self, node):
+ self.visited(node)
self.visit_sequence(node.targets)
node.value.walkabout(self)
def visit_AugAssign(self, node):
+ self.visited(node)
node.target.walkabout(self)
node.value.walkabout(self)
def visit_AnnAssign(self, node):
+ self.visited(node)
node.target.walkabout(self)
node.annotation.walkabout(self)
if node.value:
node.value.walkabout(self)
def visit_For(self, node):
+ self.visited(node)
node.target.walkabout(self)
node.iter.walkabout(self)
self.visit_sequence(node.body)
self.visit_sequence(node.orelse)
def visit_AsyncFor(self, node):
+ self.visited(node)
node.target.walkabout(self)
node.iter.walkabout(self)
self.visit_sequence(node.body)
self.visit_sequence(node.orelse)
def visit_While(self, node):
+ self.visited(node)
node.test.walkabout(self)
self.visit_sequence(node.body)
self.visit_sequence(node.orelse)
def visit_If(self, node):
+ self.visited(node)
node.test.walkabout(self)
self.visit_sequence(node.body)
self.visit_sequence(node.orelse)
def visit_With(self, node):
+ self.visited(node)
self.visit_sequence(node.items)
self.visit_sequence(node.body)
def visit_AsyncWith(self, node):
+ self.visited(node)
self.visit_sequence(node.items)
self.visit_sequence(node.body)
def visit_Raise(self, node):
+ self.visited(node)
if node.exc:
node.exc.walkabout(self)
if node.cause:
node.cause.walkabout(self)
def visit_Try(self, node):
+ self.visited(node)
self.visit_sequence(node.body)
self.visit_sequence(node.handlers)
self.visit_sequence(node.orelse)
self.visit_sequence(node.finalbody)
def visit_Assert(self, node):
+ self.visited(node)
node.test.walkabout(self)
if node.msg:
node.msg.walkabout(self)
def visit_Import(self, node):
+ self.visited(node)
self.visit_sequence(node.names)
def visit_ImportFrom(self, node):
+ self.visited(node)
self.visit_sequence(node.names)
def visit_Global(self, node):
+ self.visited(node)
pass
def visit_Nonlocal(self, node):
+ self.visited(node)
pass
def visit_Expr(self, node):
+ self.visited(node)
node.value.walkabout(self)
def visit_Pass(self, node):
+ self.visited(node)
pass
def visit_Break(self, node):
+ self.visited(node)
pass
def visit_Continue(self, node):
+ self.visited(node)
pass
def visit_BoolOp(self, node):
+ self.visited(node)
self.visit_sequence(node.values)
def visit_BinOp(self, node):
+ self.visited(node)
node.left.walkabout(self)
node.right.walkabout(self)
def visit_UnaryOp(self, node):
+ self.visited(node)
node.operand.walkabout(self)
def visit_Lambda(self, node):
+ self.visited(node)
node.args.walkabout(self)
node.body.walkabout(self)
def visit_IfExp(self, node):
+ self.visited(node)
node.test.walkabout(self)
node.body.walkabout(self)
node.orelse.walkabout(self)
def visit_Dict(self, node):
+ self.visited(node)
self.visit_sequence(node.keys)
self.visit_sequence(node.values)
def visit_Set(self, node):
+ self.visited(node)
self.visit_sequence(node.elts)
def visit_ListComp(self, node):
+ self.visited(node)
node.elt.walkabout(self)
self.visit_sequence(node.generators)
def visit_SetComp(self, node):
+ self.visited(node)
node.elt.walkabout(self)
self.visit_sequence(node.generators)
def visit_DictComp(self, node):
+ self.visited(node)
node.key.walkabout(self)
node.value.walkabout(self)
self.visit_sequence(node.generators)
def visit_GeneratorExp(self, node):
+ self.visited(node)
node.elt.walkabout(self)
self.visit_sequence(node.generators)
def visit_Await(self, node):
+ self.visited(node)
node.value.walkabout(self)
def visit_Yield(self, node):
+ self.visited(node)
if node.value:
node.value.walkabout(self)
def visit_YieldFrom(self, node):
+ self.visited(node)
node.value.walkabout(self)
def visit_Compare(self, node):
+ self.visited(node)
node.left.walkabout(self)
self.visit_sequence(node.comparators)
def visit_Call(self, node):
+ self.visited(node)
node.func.walkabout(self)
self.visit_sequence(node.args)
self.visit_sequence(node.keywords)
def visit_Num(self, node):
+ self.visited(node)
pass
def visit_Str(self, node):
+ self.visited(node)
pass
def visit_RevDBMetaVar(self, node):
+ self.visited(node)
pass
def visit_FormattedValue(self, node):
+ self.visited(node)
node.value.walkabout(self)
if node.format_spec:
node.format_spec.walkabout(self)
def visit_JoinedStr(self, node):
+ self.visited(node)
self.visit_sequence(node.values)
def visit_Bytes(self, node):
+ self.visited(node)
pass
def visit_NameConstant(self, node):
+ self.visited(node)
pass
def visit_Ellipsis(self, node):
+ self.visited(node)
pass
def visit_Constant(self, node):
+ self.visited(node)
pass
def visit_Attribute(self, node):
+ self.visited(node)
node.value.walkabout(self)
def visit_Subscript(self, node):
+ self.visited(node)
node.value.walkabout(self)
node.slice.walkabout(self)
def visit_Starred(self, node):
+ self.visited(node)
node.value.walkabout(self)
def visit_Name(self, node):
+ self.visited(node)
pass
def visit_List(self, node):
+ self.visited(node)
self.visit_sequence(node.elts)
def visit_Tuple(self, node):
+ self.visited(node)
self.visit_sequence(node.elts)
def visit_Slice(self, node):
+ self.visited(node)
if node.lower:
node.lower.walkabout(self)
if node.upper:
@@ -4534,22 +4598,27 @@
node.step.walkabout(self)
def visit_ExtSlice(self, node):
+ self.visited(node)
self.visit_sequence(node.dims)
def visit_Index(self, node):
+ self.visited(node)
node.value.walkabout(self)
def visit_comprehension(self, node):
+ self.visited(node)
node.target.walkabout(self)
node.iter.walkabout(self)
self.visit_sequence(node.ifs)
def visit_ExceptHandler(self, node):
+ self.visited(node)
if node.type:
node.type.walkabout(self)
self.visit_sequence(node.body)
def visit_arguments(self, node):
+ self.visited(node)
self.visit_sequence(node.args)
if node.vararg:
node.vararg.walkabout(self)
@@ -4560,16 +4629,20 @@
self.visit_sequence(node.defaults)
def visit_arg(self, node):
+ self.visited(node)
if node.annotation:
node.annotation.walkabout(self)
def visit_keyword(self, node):
+ self.visited(node)
node.value.walkabout(self)
def visit_alias(self, node):
+ self.visited(node)
pass
def visit_withitem(self, node):
+ self.visited(node)
node.context_expr.walkabout(self)
if node.optional_vars:
node.optional_vars.walkabout(self)
diff --git a/pypy/interpreter/astcompiler/astbuilder.py
b/pypy/interpreter/astcompiler/astbuilder.py
--- a/pypy/interpreter/astcompiler/astbuilder.py
+++ b/pypy/interpreter/astcompiler/astbuilder.py
@@ -939,7 +939,7 @@
elif comp_type == tokens.NOTEQUAL:
flufl = self.compile_info.flags &
consts.CO_FUTURE_BARRY_AS_BDFL
if flufl and comp_node.get_value() == '!=':
- self.error('invalid comparison', comp_node)
+ self.error("with Barry as BDFL, use '<>' instead of '!='",
comp_node)
elif not flufl and comp_node.get_value() == '<>':
self.error('invalid comparison', comp_node)
return ast.NotEq
diff --git a/pypy/interpreter/astcompiler/fstring.py
b/pypy/interpreter/astcompiler/fstring.py
--- a/pypy/interpreter/astcompiler/fstring.py
+++ b/pypy/interpreter/astcompiler/fstring.py
@@ -25,7 +25,7 @@
def f_constant_string(astbuilder, joined_pieces, w_u, atom_node):
add_constant_string(astbuilder, joined_pieces, w_u, atom_node)
-def f_string_compile(astbuilder, source, atom_node):
+def f_string_compile(astbuilder, source, atom_node, fstr):
# Note: a f-string is kept as a single literal up to here.
# At this point only, we recursively call the AST compiler
# on all the '{expr}' parts. The 'expr' part is not parsed
@@ -44,16 +44,44 @@
astbuilder.error("internal error: parser not available for parsing "
"the expressions inside the f-string", atom_node)
assert isinstance(source, str) # utf-8 encoded
- source = '(%s)' % source
+
+ paren_source = '(%s)' % source # to deal with whitespace at the start of
source
+
+ lineno = 0
+ column_offset = 0
+ if fstr.stnode:
+ stnode = fstr.stnode
+ lineno = stnode.get_lineno() - 1 # one-based
+ # CPython has an equivalent hack :-(
+ value = stnode.get_value()
+ if value is not None:
+ column_offset = value.find(source) + stnode.get_column()
info = pyparse.CompileInfo("<fstring>", "eval",
consts.PyCF_SOURCE_IS_UTF8 |
consts.PyCF_IGNORE_COOKIE,
optimize=astbuilder.compile_info.optimize)
parser = astbuilder.recursive_parser
- parse_tree = parser.parse_source(source, info)
- return ast_from_node(astbuilder.space, parse_tree, info,
- recursive_parser=parser)
+ parse_tree = parser.parse_source(paren_source, info)
+
+ ast = ast_from_node(astbuilder.space, parse_tree, info,
+ recursive_parser=parser)
+ fixup_fstring_positions(ast, lineno, column_offset)
+ return ast
+
+def fixup_fstring_positions(ast, line_offset, column_offset):
+ visitor = FixPosVisitor(line_offset, column_offset)
+ ast.walkabout(visitor)
+
+class FixPosVisitor(ast.GenericASTVisitor):
+ def __init__(self, line_offset, column_offset):
+ self.line_offset = line_offset
+ self.column_offset = column_offset
+
+ def visited(self, node):
+ if isinstance(node, ast.stmt) or isinstance(node, ast.expr):
+ node.lineno += self.line_offset
+ node.col_offset += self.column_offset
def unexpected_end_of_string(astbuilder, atom_node):
@@ -177,7 +205,7 @@
# Compile the expression as soon as possible, so we show errors
# related to the expression before errors related to the
# conversion or format_spec.
- expr = f_string_compile(astbuilder, s[expr_start:i], atom_node)
+ expr = f_string_compile(astbuilder, s[expr_start:i], atom_node, fstr)
assert isinstance(expr, ast.Expression)
# Check for a conversion char, if present.
@@ -345,7 +373,7 @@
child = atom_node.get_child(i)
try:
w_next = parsestring.parsestr(
- space, encoding, child.get_value())
+ space, encoding, child.get_value(), child)
if not isinstance(w_next, parsestring.W_FString):
add_constant_string(astbuilder, joined_pieces, w_next,
atom_node)
diff --git a/pypy/interpreter/astcompiler/optimize.py
b/pypy/interpreter/astcompiler/optimize.py
--- a/pypy/interpreter/astcompiler/optimize.py
+++ b/pypy/interpreter/astcompiler/optimize.py
@@ -142,6 +142,16 @@
""")
return space.pow(w_left, w_right, space.w_None)
+def _fold_lshift(space, w_left, w_right):
+ # don't constant-fold if "w_left" and "w_right" are integers and
+ # the estimated bit length of the result is unreasonably large
+ space.appexec([w_left, w_right], """(left, right):
+ if isinstance(left, int) and isinstance(right, int):
+ if left.bit_length() + right > 1000:
+ raise OverflowError
+ """)
+ return space.lshift(w_left, w_right)
+
def _fold_not(space, operand):
return space.newbool(not space.is_true(operand))
@@ -154,7 +164,7 @@
ast.FloorDiv : _binary_fold("floordiv"),
ast.Mod : _binary_fold("mod"),
ast.Pow : _fold_pow,
- ast.LShift : _binary_fold("lshift"),
+ ast.LShift : _fold_lshift,
ast.RShift : _binary_fold("rshift"),
ast.BitOr : _binary_fold("or_"),
ast.BitXor : _binary_fold("xor"),
diff --git a/pypy/interpreter/astcompiler/test/test_astbuilder.py
b/pypy/interpreter/astcompiler/test/test_astbuilder.py
--- a/pypy/interpreter/astcompiler/test/test_astbuilder.py
+++ b/pypy/interpreter/astcompiler/test/test_astbuilder.py
@@ -22,7 +22,7 @@
flags = consts.CO_FUTURE_WITH_STATEMENT
info = pyparse.CompileInfo("<test>", p_mode, flags)
tree = self.parser.parse_source(source, info)
- ast_node = ast_from_node(self.space, tree, info)
+ ast_node = ast_from_node(self.space, tree, info, self.parser)
return ast_node
def get_first_expr(self, source, p_mode=None, flags=None):
@@ -1476,3 +1476,8 @@
" bytes in position 0-1: truncated \\xXX escape")
assert exc.lineno == 2
assert exc.offset == 6
+
+ def test_fstring_lineno(self):
+ mod = self.get_ast('x=1\nf"{ x + 1}"')
+ assert mod.body[1].value.values[0].value.lineno == 2
+ assert mod.body[1].value.values[0].value.col_offset == 8
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py
b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -1474,13 +1474,14 @@
assert ops.LOAD_CONST in counts
def test_dont_fold_huge_powers(self):
- for source in (
- "2 ** 3000", # not constant-folded: too big
- "(-2) ** 3000",
+ for source, op in (
+ ("2 ** 3000", ops.BINARY_POWER), # not constant-folded: too
big
+ ("(-2) ** 3000", ops.BINARY_POWER),
+ ("5 << 1000", ops.BINARY_LSHIFT),
):
source = 'def f(): %s' % source
counts = self.count_instructions(source)
- assert ops.BINARY_POWER in counts
+ assert op in counts
for source in (
"2 ** 2000", # constant-folded
diff --git a/pypy/interpreter/astcompiler/tools/asdl_py.py
b/pypy/interpreter/astcompiler/tools/asdl_py.py
--- a/pypy/interpreter/astcompiler/tools/asdl_py.py
+++ b/pypy/interpreter/astcompiler/tools/asdl_py.py
@@ -341,6 +341,9 @@
def visitModule(self, mod):
self.emit("class GenericASTVisitor(ASTVisitor):")
self.emit("")
+ self.emit("def visited(self, node):", 1)
+ self.emit("pass # base implementation", 2)
+ self.emit("")
super(GenericASTVisitorVisitor, self).visitModule(mod)
self.emit("")
@@ -357,6 +360,7 @@
def make_visitor(self, name, fields):
self.emit("def visit_%s(self, node):" % (name,), 1)
+ self.emit("self.visited(node)", 2)
have_body = False
for field in fields:
if self.visitField(field):
diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py
--- a/pypy/interpreter/generator.py
+++ b/pypy/interpreter/generator.py
@@ -785,4 +785,8 @@
# StopIteration, signalling that this 'aclose()' await
# is done.
raise OperationError(space.w_StopIteration, space.w_None)
+ if e.match(space, space.w_GeneratorExit):
+ if self.w_exc_type is None:
+ # Ignore this error.
+ raise OperationError(space.w_StopIteration, space.w_None)
raise e
diff --git a/pypy/interpreter/pyparser/parsestring.py
b/pypy/interpreter/pyparser/parsestring.py
--- a/pypy/interpreter/pyparser/parsestring.py
+++ b/pypy/interpreter/pyparser/parsestring.py
@@ -7,14 +7,15 @@
class W_FString(W_Root):
- def __init__(self, unparsed, raw_mode):
+ def __init__(self, unparsed, raw_mode, stnode):
assert isinstance(unparsed, str) # utf-8 encoded string
self.unparsed = unparsed # but the quotes are removed
self.raw_mode = raw_mode
self.current_index = 0 # for astcompiler.fstring
+ self.stnode = stnode
-def parsestr(space, encoding, s):
+def parsestr(space, encoding, s, stnode=None):
"""Parses a string or unicode literal, and return usually
a wrapped value. If we get an f-string, then instead return
an unparsed but unquoted W_FString instance.
@@ -88,7 +89,7 @@
if unicode_literal and not rawmode: # XXX Py_UnicodeFlag is ignored for now
assert 0 <= ps <= q
if saw_f:
- return W_FString(s[ps:q], rawmode)
+ return W_FString(s[ps:q], rawmode, stnode)
if encoding is None:
substr = s[ps:q]
else:
@@ -112,7 +113,7 @@
if not unicode_literal:
return space.newbytes(substr)
elif saw_f:
- return W_FString(substr, rawmode)
+ return W_FString(substr, rawmode, stnode)
else:
v = unicodehelper.str_decode_utf8(substr, 'strict', True, None)
return space.newtext(*v)
diff --git a/pypy/interpreter/test/apptest_coroutine.py
b/pypy/interpreter/test/apptest_coroutine.py
--- a/pypy/interpreter/test/apptest_coroutine.py
+++ b/pypy/interpreter/test/apptest_coroutine.py
@@ -202,6 +202,7 @@
import gc, warnings # XXX: importing warnings is expensive untranslated
async def foobaz():
pass
+ gc.collect() # emit warnings from unrelated older tests
with warnings.catch_warnings(record=True) as l:
foobaz()
gc.collect()
diff --git a/pypy/interpreter/test/apptest_pyframe.py
b/pypy/interpreter/test/apptest_pyframe.py
--- a/pypy/interpreter/test/apptest_pyframe.py
+++ b/pypy/interpreter/test/apptest_pyframe.py
@@ -13,7 +13,8 @@
import sys
f = sys._getframe()
assert f.f_globals is globals()
- pytest.raises(AttributeError, "f.f_globals = globals()")
+ with pytest.raises(AttributeError):
+ f.f_globals = globals()
def test_f_builtins():
import sys, builtins
diff --git a/pypy/interpreter/test/test_compiler.py
b/pypy/interpreter/test/test_compiler.py
--- a/pypy/interpreter/test/test_compiler.py
+++ b/pypy/interpreter/test/test_compiler.py
@@ -706,31 +706,26 @@
def test_dont_inherit_flag(self):
# this test checks that compile() don't inherit the __future__ flags
- # of the hosting code. However, in Python3 we don't have any
- # meaningful __future__ flag to check that (they are all enabled). The
- # only candidate could be barry_as_FLUFL, but it's not implemented yet
- # (and not sure it'll ever be)
- py.test.skip("we cannot actually check the result of this test (see
comment)")
+ # of the hosting code.
space = self.space
s1 = str(py.code.Source("""
- from __future__ import division
- exec(compile('x = 1/2', '?', 'exec', 0, 1))
+ from __future__ import barry_as_FLUFL
+ # not a syntax error inside the exec!
+ exec(compile('x = 1 != 2', '?', 'exec', 0, 1))
"""))
w_result = space.appexec([space.wrap(s1)], """(s1):
ns = {}
exec(s1, ns)
return ns['x']
""")
- assert space.float_w(w_result) == 0
+ assert space.is_true(w_result)
def test_dont_inherit_across_import(self):
- # see the comment for test_dont_inherit_flag
- py.test.skip("we cannot actually check the result of this test (see
comment)")
from rpython.tool.udir import udir
- udir.join('test_dont_inherit_across_import.py').write('x = 1/2\n')
+ udir.join('test_dont_inherit_across_import.py').write('x = 1 != 2\n')
space = self.space
s1 = str(py.code.Source("""
- from __future__ import division
+ from __future__ import barry_as_FLUFL
from test_dont_inherit_across_import import x
"""))
w_result = space.appexec([space.wrap(str(udir)), space.wrap(s1)],
@@ -738,13 +733,14 @@
import sys
copy = sys.path[:]
sys.path.insert(0, udir)
+ ns = {}
try:
- exec s1
+ exec(s1, ns)
finally:
sys.path[:] = copy
- return x
+ return ns['x']
""")
- assert space.float_w(w_result) == 0
+ assert space.is_true(w_result)
def test_filename_in_syntaxerror(self):
e = py.test.raises(OperationError, self.compiler.compile, """if 1:
@@ -917,9 +913,11 @@
code = "from __future__ import barry_as_FLUFL; 2 {0} 3"
compile(code.format('<>'), '<BDFL test>', 'exec',
__future__.CO_FUTURE_BARRY_AS_BDFL)
- raises(SyntaxError, compile, code.format('!='),
+ with raises(SyntaxError) as excinfo:
+ compile(code.format('!='),
'<FLUFL test>', 'exec',
__future__.CO_FUTURE_BARRY_AS_BDFL)
+ assert excinfo.value.msg == "with Barry as BDFL, use '<>' instead of
'!='"
def test_guido_as_bdfl(self):
# from test_flufl.py :-)
diff --git a/pypy/interpreter/test/test_function.py
b/pypy/interpreter/test/test_function.py
--- a/pypy/interpreter/test/test_function.py
+++ b/pypy/interpreter/test/test_function.py
@@ -160,7 +160,8 @@
return 41
assert f() == 42
assert g() == 41
- raises(TypeError, "f.__code__ = 1")
+ with raises(TypeError):
+ f.__code__ = 1
f.__code__ = g.__code__
assert f() == 41
def get_h(f=f):
@@ -168,14 +169,17 @@
return f() # a closure
return h
h = get_h()
- raises(ValueError, "f.__code__ = h.__code__")
+ with raises(ValueError):
+ f.__code__ = h.__code__
@pytest.mark.skipif("config.option.runappdirect")
def test_write_code_builtin_forbidden(self):
def f(*args):
return 42
- raises(TypeError, "dir.__code__ = f.__code__")
- raises(TypeError, "list.append.__code__ = f.__code__")
+ with raises(TypeError):
+ dir.__code__ = f.__code__
+ with raises(TypeError):
+ list.append.__code__ = f.__code__
def test_set_module_to_name_eagerly(self):
skip("fails on PyPy but works on CPython. Unsure we want to care")
@@ -313,19 +317,10 @@
def func(self, **kw):
return self, kw
func = A().func
-
- # don't want the extra argument passing of raises
- try:
+ with raises(TypeError):
func(self=23)
- assert False
- except TypeError:
- pass
-
- try:
+ with raises(TypeError):
func(**{'self': 23})
- assert False
- except TypeError:
- pass
def test_kwargs_confusing_name(self):
def func(self): # 'self' conflicts with the interp-level
diff --git a/pypy/interpreter/test/test_nestedscope.py
b/pypy/interpreter/test/test_nestedscope.py
--- a/pypy/interpreter/test/test_nestedscope.py
+++ b/pypy/interpreter/test/test_nestedscope.py
@@ -99,7 +99,8 @@
x = 1
g = f()
- raises(ValueError, "g.__closure__[0].cell_contents")
+ with raises(ValueError):
+ g.__closure__[0].cell_contents
def test_compare_cells(self):
def f(n):
diff --git a/pypy/interpreter/test/test_pyframe.py
b/pypy/interpreter/test/test_pyframe.py
--- a/pypy/interpreter/test/test_pyframe.py
+++ b/pypy/interpreter/test/test_pyframe.py
@@ -62,7 +62,7 @@
jump_out_of_block_forwards.jump = (3, 5)
jump_out_of_block_forwards.output = [2, 5]
- #run_test(jump_out_of_block_forwards)
+ run_test(jump_out_of_block_forwards)
def jump_out_of_block_backwards(output):
output.append(1)
diff --git a/pypy/interpreter/test/test_raise.py
b/pypy/interpreter/test/test_raise.py
--- a/pypy/interpreter/test/test_raise.py
+++ b/pypy/interpreter/test/test_raise.py
@@ -2,9 +2,8 @@
class AppTestRaise:
def test_arg_as_string(self):
- def f():
+ with raises(TypeError):
raise "test"
- raises(TypeError, f)
def test_control_flow(self):
try:
@@ -36,9 +35,8 @@
assert isinstance(e, IndexError)
def test_raise_cls(self):
- def f():
+ with raises(IndexError):
raise IndexError
- raises(IndexError, f)
def test_raise_cls_catch(self):
def f(r):
@@ -46,7 +44,8 @@
raise r
except LookupError:
return 1
- raises(Exception, f, Exception)
+ with raises(Exception):
+ f(Exception)
assert f(IndexError) == 1
def test_raise_wrong(self):
@@ -99,7 +98,7 @@
assert sys.exc_info() == (None, None, None)
def test_reraise_1(self):
- raises(IndexError, """
+ with raises(IndexError):
import sys
try:
raise ValueError
@@ -109,10 +108,10 @@
finally:
assert sys.exc_info()[0] is IndexError
raise
- """)
+
def test_reraise_2(self):
- raises(IndexError, """
+ with raises(IndexError):
def foo():
import sys
assert sys.exc_info()[0] is IndexError
@@ -124,10 +123,10 @@
raise IndexError
finally:
foo()
- """)
+
def test_reraise_3(self):
- raises(IndexError, """
+ with raises(IndexError):
def spam():
import sys
try:
@@ -142,7 +141,6 @@
raise IndexError
finally:
spam()
- """)
def test_reraise_4(self):
import sys
@@ -156,7 +154,7 @@
assert ok
def test_reraise_5(self):
- raises(IndexError, """
+ with raises(IndexError):
import sys
try:
raise ValueError
@@ -170,17 +168,16 @@
finally:
assert sys.exc_info()[0] is IndexError
assert sys.exc_info()[2].tb_next is some_traceback
- """)
def test_nested_reraise(self):
- raises(TypeError, """
+ with raises(TypeError):
def nested_reraise():
raise
try:
raise TypeError("foo")
except:
nested_reraise()
- """)
+
def test_with_reraise_1(self):
class Context:
@@ -196,7 +193,8 @@
with Context():
pass
raise
- raises(ValueError, "fn()")
+ with raises(ValueError):
+ fn()
def test_with_reraise_2(self):
@@ -213,23 +211,20 @@
with Context():
raise KeyError("caught")
raise
- raises(ValueError, "fn()")
+ with raises(ValueError):
+ fn()
def test_userclass(self):
# new-style classes can't be raised unless they inherit from
# BaseException
-
class A(object):
def __init__(self, x=None):
self.x = x
-
- def f():
+
+ with raises(TypeError):
raise A
- raises(TypeError, f)
-
- def f():
+ with raises(TypeError):
raise A(42)
- raises(TypeError, f)
def test_userclass_catch(self):
# classes can't be caught unless they inherit from BaseException
@@ -259,7 +254,7 @@
def test_catch_tuple(self):
class A(Exception):
pass
-
+
try:
raise ValueError
except (ValueError, A):
@@ -307,7 +302,9 @@
class MyException(Exception):
def __new__(cls, *args):
return object()
- raises(TypeError, "raise MyException")
+
+ with raises(TypeError):
+ raise MyException
def test_with_exit_True(self):
class X:
diff --git a/pypy/interpreter/test/test_syntax.py
b/pypy/interpreter/test/test_syntax.py
--- a/pypy/interpreter/test/test_syntax.py
+++ b/pypy/interpreter/test/test_syntax.py
@@ -344,9 +344,6 @@
class AppTestWith:
def test_with_simple(self):
-
- s = """
-if 1:
class Context:
def __init__(self):
self.calls = list()
@@ -360,78 +357,28 @@
acontext = Context()
with acontext:
pass
- """
- ns = {}
- exec(s, ns)
- acontext = ns['acontext']
assert acontext.calls == '__enter__ __exit__'.split()
def test_compound_with(self):
- s = """class Context:
- def __init__(self, var):
- self.record = []
- self.var = var
- def __enter__(self):
- self.record.append(("__enter__", self.var))
- return self.var
- def __exit__(self, tp, value, tb):
- self.record.append(("__exit__", self.var))
-c1 = Context("blah")
-c2 = Context("bling")
-with c1 as v1, c2 as v2:
- pass
- """
- ns = {}
- exec(s, ns)
- assert ns["v1"] == "blah"
- assert ns["v2"] == "bling"
- assert ns["c1"].record == [("__enter__", "blah"), ("__exit__", "blah")]
- assert ns["c2"].record == [("__enter__", "bling"),
- ("__exit__", "bling")]
-
-
- def test_start_with_blank_line(self):
- s = """
-if 1:
class Context:
- def __init__(self):
- self.calls = list()
-
+ def __init__(self, var):
+ self.record = []
+ self.var = var
def __enter__(self):
- self.calls.append('__enter__')
-
- def __exit__(self, exc_type, exc_value, exc_tb):
- self.calls.append('__exit__')
-
- acontext = Context()
- with acontext:
+ self.record.append(("__enter__", self.var))
+ return self.var
+ def __exit__(self, tp, value, tb):
+ self.record.append(("__exit__", self.var))
+ c1 = Context("blah")
+ c2 = Context("bling")
+ with c1 as v1, c2 as v2:
pass
-"""
- ns = {}
- exec(s, ns)
- acontext = ns['acontext']
- assert acontext.calls == '__enter__ __exit__'.split()
-
- def test_raw_doc_string(self):
- s = """r'doc'
-class Context(object):
- def __enter__(self):
- global enter
- enter = True
- def __exit__(self, *exc):
- global exit
- exit = True
-with Context() as w:
- pass"""
- ns = {}
- exec(s, ns)
- assert ns['enter']
- assert ns['exit']
+ assert v1 == "blah"
+ assert v2 == "bling"
+ assert c1.record == [("__enter__", "blah"), ("__exit__", "blah")]
+ assert c2.record == [("__enter__", "bling"), ("__exit__", "bling")]
def test_with_as_var(self):
-
- s = """
-if 1:
class Context:
def __init__(self):
self.calls = list()
@@ -448,17 +395,10 @@
with acontextfact as avar:
avar.append('__body__')
pass
- """
- ns = {}
- exec(s, ns)
- acontextfact = ns['acontextfact']
assert acontextfact.exit_params == (None, None, None)
assert acontextfact.calls == '__enter__ __body__ __exit__'.split()
def test_with_raise_exception(self):
-
- s = """
-if 1:
class Context:
def __init__(self):
self.calls = list()
@@ -482,20 +422,12 @@
pass
else:
raise AssertionError('With did not raise RuntimeError')
- """
- ns = {}
- exec(s, ns)
- acontextfact = ns['acontextfact']
- error = ns['error']
assert acontextfact.calls == '__enter__ __body__ __exit__'.split()
assert acontextfact.exit_params[0:2] == (RuntimeError, error)
import types
assert isinstance(acontextfact.exit_params[2], types.TracebackType)
def test_with_swallow_exception(self):
-
- s = """
-if 1:
class Context:
def __init__(self):
self.calls = list()
@@ -515,11 +447,6 @@
avar.append('__body__')
raise error
avar.append('__after_raise__')
- """
- ns = {}
- exec(s, ns)
- acontextfact = ns['acontextfact']
- error = ns['error']
assert acontextfact.calls == '__enter__ __body__ __exit__'.split()
assert acontextfact.exit_params[0:2] == (RuntimeError, error)
import types
@@ -544,9 +471,6 @@
assert c.calls == ['exit']
def test_with_break(self):
-
- s = """
-if 1:
class Context:
def __init__(self):
self.calls = list()
@@ -568,17 +492,10 @@
avar.append('__after_break__')
else:
raise AssertionError('Break failed with With, reached else clause')
- """
- ns = {}
- exec(s, ns)
- acontextfact = ns['acontextfact']
assert acontextfact.calls == '__enter__ __body__ __exit__'.split()
assert acontextfact.exit_params == (None, None, None)
def test_with_continue(self):
-
- s = """
-if 1:
class Context:
def __init__(self):
self.calls = list()
@@ -600,16 +517,10 @@
avar.append('__after_continue__')
else:
avar.append('__continue__')
- """
- ns = {}
- exec(s, ns)
- acontextfact = ns['acontextfact']
assert acontextfact.calls == '__enter__ __body__ __exit__
__continue__'.split()
assert acontextfact.exit_params == (None, None, None)
def test_with_return(self):
- s = """
-if 1:
class Context:
def __init__(self):
self.calls = list()
@@ -630,28 +541,16 @@
return '__return__'
avar.append('__after_return__')
acontextfact.calls.append(g(acontextfact))
- """
- ns = {}
- exec(s, ns)
- acontextfact = ns['acontextfact']
assert acontextfact.calls == '__enter__ __body__ __exit__
__return__'.split()
assert acontextfact.exit_params == (None, None, None)
def test_with_as_keyword(self):
- try:
+ with raises(SyntaxError):
exec("with = 9")
- except SyntaxError:
- pass
- else:
- assert False, 'Assignment to with did not raise SyntaxError'
def test_with_as_keyword_compound(self):
- try:
+ with raises(SyntaxError):
exec("from __future__ import generators, with_statement\nwith = 9")
- except SyntaxError:
- pass
- else:
- assert False, 'Assignment to with did not raise SyntaxError'
def test_missing_as_SyntaxError(self):
snippets = [
@@ -662,21 +561,10 @@
pass
"""]
for snippet in snippets:
- try:
+ with raises(SyntaxError):
exec(snippet)
- except SyntaxError:
- pass
- else:
- assert False, "%s: did not raise SyntaxError" % snippet
- def test_with_propagate_compileflag(self):
- s = """
-if 1:
- compile('''with x:
- pass''', '', 'exec')
- """
- exec(s)
class AppTestFunctionAnnotations:
@@ -707,7 +595,6 @@
pass
f1()
"""
-
class AppTestSyntaxError:
def test_tokenizer_error_location(self):
@@ -752,7 +639,8 @@
# -*- coding: uft-8 -*-
pass
"""
- raises(SyntaxError, exec, program)
+ with raises(SyntaxError):
+ exec(program)
'''
def test_exception_target_in_nested_scope(self):
@@ -798,24 +686,3 @@
raise AssertionError("should have raised")
"""
-
-if __name__ == '__main__':
- # only to check on top of CPython (you need 2.4)
- from py.test import raises
- for s in VALID:
- try:
- compile(s, '?', 'exec')
- except Exception as e:
- print '-'*20, 'FAILED TO COMPILE:', '-'*20
- print s
- print '%s: %s' % (e.__class__, e)
- print '-'*60
- for s in INVALID:
- try:
- raises(SyntaxError, compile, s, '?', 'exec')
- except Exception as e:
- print '-'*20, 'UNEXPECTEDLY COMPILED:', '-'*20
- print s
- print '%s: %s' % (e.__class__, e)
- print '-'*60
-
diff --git a/pypy/interpreter/test/test_unicodehelper.py
b/pypy/interpreter/test/test_unicodehelper.py
--- a/pypy/interpreter/test/test_unicodehelper.py
+++ b/pypy/interpreter/test/test_unicodehelper.py
@@ -1,7 +1,7 @@
import pytest
from pypy.interpreter.unicodehelper import (
- utf8_encode_utf_8, decode_utf8sp,
+ utf8_encode_utf_8, decode_utf8sp, ErrorHandlerError
)
@@ -21,7 +21,11 @@
called with a start and stop position of the full surrogate
pair (new behavior in python3.6)
"""
- u = u"\udc80\ud800\udfff"
+ # /--surrogate pair--\
+ # \udc80 \ud800 \udfff
+ b = "\xed\xb2\x80\xed\xa0\x80\xed\xbf\xbf"
+
+ calls = []
def errorhandler(errors, encoding, msg, s, start, end):
"""
@@ -32,31 +36,23 @@
2. the second time, the characters will be 0xD800 and 0xDFFF, since
that is a valid surrogate pair.
"""
- assert s[start:end] in [u'\udc80', u'\uD800\uDFFF']
- return '', end, 'b'
+ calls.append(s.decode("utf-8")[start:end])
+ return 'abc', end, 'b'
- utf8_encode_utf_8(
- u, 'strict',
+ res = utf8_encode_utf_8(
+ b, 'strict',
errorhandler=errorhandler,
allow_surrogates=False
)
+ assert res == "abcabc"
+ assert calls == [u'\udc80', u'\uD800\uDFFF']
def test_bad_error_handler():
- u = u"\udc80\ud800\udfff"
+ b = u"\udc80\ud800\udfff".encode("utf-8")
+ def errorhandler(errors, encoding, msg, s, start, end):
+ return '', start, 'b' # returned index is too small
- def errorhandler(errors, encoding, msg, s, start, end):
- """
- This handler will be called twice, so asserting both times:
-
- 1. the first time, 0xDC80 will be handled as a single surrogate,
- since it is a standalone character and an invalid surrogate.
- 2. the second time, the characters will be 0xD800 and 0xDFFF, since
- that is a valid surrogate pair.
- """
- assert s[start:end] in [u'\udc80', u'\uD800\uDFFF']
- return '', start, 'b'
-
- assert pytest.raises(Exception, utf8_encode_utf_8, u, 'strict',
+ pytest.raises(ErrorHandlerError, utf8_encode_utf_8, b, 'strict',
errorhandler=errorhandler, allow_surrogates=False)
def test_decode_utf8sp():
diff --git a/pypy/interpreter/unicodehelper.py
b/pypy/interpreter/unicodehelper.py
--- a/pypy/interpreter/unicodehelper.py
+++ b/pypy/interpreter/unicodehelper.py
@@ -31,7 +31,7 @@
# Fast version of the "strict" errors handler.
def raise_unicode_exception_encode(errors, encoding, msg, utf8,
startingpos, endingpos):
- u_len = rutf8.get_utf8_length(utf8)
+ u_len = rutf8.codepoints_in_utf8(utf8)
raise OperationError(space.w_UnicodeEncodeError,
space.newtuple([space.newtext(encoding),
space.newutf8(utf8, u_len),
@@ -120,12 +120,6 @@
return encode_object(space, w_data, encoding, errors)
-def _has_surrogate(u):
- for c in u:
- if 0xD800 <= ord(c) <= 0xDFFF:
- return True
- return False
-
# These functions take and return unwrapped rpython strings
def decode_unicode_escape(space, string):
from pypy.module._codecs import interp_codecs
@@ -221,23 +215,35 @@
size = len(s)
if size == 0:
return ''
+
+ # two fast paths
+ if allow_surrogates:
+ # already valid utf-8 with surrogates, surrogates are allowed, so just
+ # return
+ return s
+ if not rutf8.has_surrogates(s):
+ # already valid utf-8 and doesn't contain surrogates, so we don't need
+ # to do anything
+ return s
+ # annoying slow path
+ return _utf8_encode_utf_8_deal_with_surrogates(s, errors, errorhandler)
+
+def _utf8_encode_utf_8_deal_with_surrogates(s, errors, errorhandler):
pos = 0
upos = 0
+ size = len(s)
result = StringBuilder(size)
while pos < size:
try:
- lgt = rutf8.check_utf8(s, allow_surrogates=allow_surrogates,
start=pos)
- if pos == 0:
- # fast path
- return s
- for ch in s[pos:]:
- result.append(ch)
+ rutf8.check_utf8(s, allow_surrogates=False, start=pos)
+ # otherwise the fast path above would have triggered
+ assert pos != 0
+ result.append_slice(s, pos, len(s))
break
except rutf8.CheckError as e:
end = e.pos
assert end >= 0
- for ch in s[pos:end]:
- result.append(ch)
+ result.append_slice(s, pos, end)
upos += rutf8.codepoints_in_utf8(s, start=pos, end=end)
pos = end
# Try to get collect surrogates in one pass
@@ -254,11 +260,13 @@
res, newindex, rettype = errorhandler(errors, 'utf-8',
'surrogates not allowed', s, upos, upos + delta)
if rettype == 'u':
- for cp in rutf8.Utf8StringIterator(res):
- result.append(chr(cp))
- else:
- for ch in res:
- result.append(ch)
+ try:
+ rutf8.check_ascii(res)
+ except rutf8.CheckError:
+ # this is a weird behaviour of CPython, but it's what
happens
+ errorhandler("strict", 'utf-8', 'surrogates not allowed',
s, upos, upos + delta)
+ assert 0, "unreachable"
+ result.append(res)
if newindex <= upos:
raise ErrorHandlerError(newindex, upos)
upos = newindex
diff --git a/pypy/module/__builtin__/compiling.py
b/pypy/module/__builtin__/compiling.py
--- a/pypy/module/__builtin__/compiling.py
+++ b/pypy/module/__builtin__/compiling.py
@@ -133,11 +133,11 @@
if not space.ismapping_w(w_namespace):
if isclass:
raise oefmt(space.w_TypeError,
- "%N.__prepare__ must return a mapping, not %T",
+ "%N.__prepare__() must return a mapping, not %T",
w_meta, w_namespace)
else:
raise oefmt(space.w_TypeError,
- "<metaclass>.__prepare__ must return a mapping, not %T",
+ "<metaclass>.__prepare__() must return a mapping, not %T",
w_namespace)
code = w_func.getcode()
diff --git a/pypy/module/__builtin__/test/apptest_compile.py
b/pypy/module/__builtin__/test/apptest_compile.py
--- a/pypy/module/__builtin__/test/apptest_compile.py
+++ b/pypy/module/__builtin__/test/apptest_compile.py
@@ -179,3 +179,20 @@
assert 'module_doc' not in marshalled
assert 'func_doc' not in marshalled
assert 'class_doc' not in marshalled
+
+def test_build_class():
+ """Test error message bad __prepare__"""
+
+ class BadMeta(type):
+ @classmethod
+ def __prepare__(*args):
+ return None
+
+ def func():
+ class Foo(metaclass=BadMeta):
+ pass
+
+ excinfo = raises(TypeError, func)
+ assert str(excinfo.value) == (
+ r"BadMeta.__prepare__() must return a mapping, not NoneType"
+ )
diff --git a/pypy/module/_cffi_backend/test/test_recompiler.py
b/pypy/module/_cffi_backend/test/test_recompiler.py
--- a/pypy/module/_cffi_backend/test/test_recompiler.py
+++ b/pypy/module/_cffi_backend/test/test_recompiler.py
@@ -75,8 +75,12 @@
args_w = [space.wrap(module_name), space.wrap(so_file)]
w_res = space.appexec(args_w, """(modulename, filename):
- import imp
- mod = imp.load_dynamic(modulename, filename)
+ import _imp
+ class Spec: pass
+ spec = Spec()
+ spec.name = modulename
+ spec.origin = filename
+ mod = _imp.create_dynamic(spec)
assert mod.__name__ == modulename
return (mod.ffi, mod.lib)
""")
diff --git a/pypy/module/_codecs/interp_codecs.py
b/pypy/module/_codecs/interp_codecs.py
--- a/pypy/module/_codecs/interp_codecs.py
+++ b/pypy/module/_codecs/interp_codecs.py
@@ -715,6 +715,8 @@
@unwrap_spec(errors='text_or_none')
def utf_8_encode(space, w_obj, errors="strict"):
utf8, lgt = space.utf8_len_w(w_obj)
+ if lgt == len(utf8): # ascii
+ return space.newtuple([space.newbytes(utf8), space.newint(lgt)])
if errors is None:
errors = 'strict'
state = space.fromcache(CodecState)
diff --git a/pypy/module/_codecs/test/test_codecs.py
b/pypy/module/_codecs/test/test_codecs.py
--- a/pypy/module/_codecs/test/test_codecs.py
+++ b/pypy/module/_codecs/test/test_codecs.py
@@ -1041,6 +1041,14 @@
# CPython raises OverflowError here
raises((IndexError, OverflowError), b'apple\x92ham\x93spam'.decode,
'utf-8', errors)
+ def test_badhandler_returns_unicode(self):
+ import codecs
+ import sys
+ errors = 'test.badhandler_unicode'
+ codecs.register_error(errors, lambda x: (chr(100000), x.end))
+ # CPython raises OverflowError here
+ raises(UnicodeEncodeError, u'\udc80\ud800\udfff'.encode, 'utf-8',
errors)
+
def test_unicode_internal(self):
import codecs
import sys
diff --git a/pypy/module/_io/interp_iobase.py b/pypy/module/_io/interp_iobase.py
--- a/pypy/module/_io/interp_iobase.py
+++ b/pypy/module/_io/interp_iobase.py
@@ -56,7 +56,7 @@
# `__IOBase_closed` and call flush() by itself, but it is redundant
# with whatever behaviour a non-trivial derived class will implement.
self.space = space
- self.w_dict = space.newdict()
+ self.w_dict = space.newdict(instance=True)
self.__IOBase_closed = False
if add_to_autoflusher:
get_autoflusher(space).add(self)
diff --git a/pypy/module/_io/interp_stringio.py
b/pypy/module/_io/interp_stringio.py
--- a/pypy/module/_io/interp_stringio.py
+++ b/pypy/module/_io/interp_stringio.py
@@ -1,4 +1,4 @@
-from rpython.rlib.rutf8 import get_utf8_length, next_codepoint_pos
+from rpython.rlib.rutf8 import codepoints_in_utf8, next_codepoint_pos
from pypy.interpreter.error import OperationError, oefmt
from pypy.interpreter.typedef import (
@@ -98,7 +98,7 @@
return result
def write(self, string):
- length = get_utf8_length(string)
+ length = codepoints_in_utf8(string)
if self.pos + length > len(self.data):
self.resize(self.pos + length)
pos = 0
@@ -173,7 +173,7 @@
if readnl is None:
w_readnl = space.w_None
else:
- w_readnl = space.str(space.newutf8(readnl,
get_utf8_length(readnl))) # YYY
+ w_readnl = space.str(space.newutf8(readnl,
codepoints_in_utf8(readnl))) # YYY
return space.newtuple([
w_initialval, w_readnl, space.newint(self.buf.pos), w_dict
])
@@ -239,7 +239,7 @@
w_decoded = space.call_method(
w_decoded, "replace",
space.newtext("\n"),
- space.newutf8(writenl, get_utf8_length(writenl)),
+ space.newutf8(writenl, codepoints_in_utf8(writenl)),
)
string = space.utf8_w(w_decoded)
if string:
@@ -251,7 +251,7 @@
self._check_closed(space)
size = convert_size(space, w_size)
v = self.buf.read(size)
- lgt = get_utf8_length(v)
+ lgt = codepoints_in_utf8(v)
return space.newutf8(v, lgt)
def readline_w(self, space, w_limit=None):
@@ -266,7 +266,7 @@
else:
newline = self.readnl
result = self.buf.readline(newline, limit)
- resultlen = get_utf8_length(result)
+ resultlen = codepoints_in_utf8(result)
return space.newutf8(result, resultlen)
@@ -305,7 +305,7 @@
def getvalue_w(self, space):
self._check_closed(space)
v = self.buf.getvalue()
- lgt = get_utf8_length(v)
+ lgt = codepoints_in_utf8(v)
return space.newutf8(v, lgt)
def readable_w(self, space):
diff --git a/pypy/module/_io/interp_textio.py b/pypy/module/_io/interp_textio.py
--- a/pypy/module/_io/interp_textio.py
+++ b/pypy/module/_io/interp_textio.py
@@ -12,7 +12,7 @@
from rpython.rlib.rbigint import rbigint
from rpython.rlib.rstring import StringBuilder
from rpython.rlib.rutf8 import (check_utf8, next_codepoint_pos,
- codepoints_in_utf8, get_utf8_length,
+ codepoints_in_utf8, codepoints_in_utf8,
Utf8StringBuilder)
@@ -308,13 +308,18 @@
class DecodeBuffer(object):
- def __init__(self, text=None):
+ def __init__(self, text=None, ulen=-1):
+ # self.text is a valid utf-8 string
+ if text is not None:
+ assert ulen >= 0
self.text = text
self.pos = 0
self.upos = 0
+ self.ulen = ulen
def set(self, space, w_decoded):
check_decoded(space, w_decoded)
+ self.ulen = space.len_w(w_decoded)
self.text = space.utf8_w(w_decoded)
self.pos = 0
self.upos = 0
@@ -323,12 +328,14 @@
self.text = None
self.pos = 0
self.upos = 0
+ self.ulen = -1
def get_chars(self, size):
+ """ returns a tuple (utf8, lgt) """
if self.text is None or size == 0:
- return ""
+ return "", 0
- lgt = codepoints_in_utf8(self.text)
+ lgt = self.ulen
available = lgt - self.upos
if size < 0 or size > available:
size = available
@@ -337,7 +344,7 @@
if self.pos > 0 or size < available:
start = self.pos
pos = start
- for i in range(size):
+ for i in range(size):
pos = next_codepoint_pos(self.text, pos)
self.upos += 1
assert start >= 0
@@ -348,8 +355,9 @@
chars = self.text
self.pos = len(self.text)
self.upos = lgt
+ size = lgt
- return chars
+ return chars, size
def has_data(self):
return (self.text is not None and not self.exhausted())
@@ -386,22 +394,22 @@
limit = sys.maxint
scanned = 0
while scanned < limit:
- try:
- ch = self.next_char()
- scanned += 1
- except StopIteration:
+ if self.exhausted():
return False
+ ch = self.text[self.pos]
+ self._advance_codepoint()
+ scanned += 1
if ch == '\n':
return True
if ch == '\r':
if scanned >= limit:
return False
- try:
- ch = self.peek_char()
- except StopIteration:
+ if self.exhausted():
+ # don't split potential \r\n
return False
+ ch = self.text[self.pos]
if ch == '\n':
- self.next_char()
+ self._advance_codepoint()
return True
else:
return True
@@ -412,39 +420,66 @@
limit = sys.maxint
scanned = 0
while scanned < limit:
- try:
- ch = self.next_char()
- except StopIteration:
+ if self.exhausted():
return False
+ ch = self.text[self.pos]
+ self._advance_codepoint()
scanned += 1
if ch == '\r':
if scanned >= limit:
return False
- try:
- if self.peek_char() == '\n':
- self.next_char()
- return True
- except StopIteration:
- # This is the tricky case: we found a \r right at the end
+ if self.exhausted():
+ # This is the tricky case: we found a \r right at the end,
+ # un-consume it
self.pos -= 1
self.upos -= 1
return False
+ if self.text[self.pos] == '\n':
+ self._advance_codepoint()
+ return True
return False
def find_char(self, marker, limit):
+ # only works for ascii markers!
+ assert 0 <= ord(marker) < 128
+ # ascii fast path
+ if self.ulen == len(self.text):
+ end = len(self.text)
+ if limit >= 0:
+ end = min(end, self.pos + limit)
+ pos = self.pos
+ assert pos >= 0
+ assert end >= 0
+ pos = self.text.find(marker, pos, end)
+ if pos >= 0:
+ self.pos = self.upos = pos + 1
+ return True
+ else:
+ self.pos = self.upos = end
+ return False
+
if limit < 0:
_______________________________________________
pypy-commit mailing list
[email protected]
https://mail.python.org/mailman/listinfo/pypy-commit
