[pypy-commit] pypy pyparser-improvements-3: merge default
Author: Carl Friedrich Bolz-Tereick
Branch: pyparser-improvements-3
Changeset: r94729:37acacd15a8b
Date: 2018-06-06 14:27 +0200
http://bitbucket.org/pypy/pypy/changeset/37acacd15a8b/
Log:merge default
diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py
b/pypy/interpreter/astcompiler/test/test_compiler.py
--- a/pypy/interpreter/astcompiler/test/test_compiler.py
+++ b/pypy/interpreter/astcompiler/test/test_compiler.py
@@ -27,7 +27,7 @@
generator._resolve_block_targets(blocks)
return generator, blocks
-class TestCompiler:
+class BaseTestCompiler:
"""These tests compile snippets of code and check them by
running them with our own interpreter. These are thus not
completely *unit* tests, but given that our interpreter is
@@ -74,6 +74,9 @@
def error_test(self, source, exc_type):
py.test.raises(exc_type, self.simple_test, source, None, None)
+
+class TestCompiler(BaseTestCompiler):
+
def test_issue_713(self):
func = "def f(_=2): return (_ if _ else _) if False else _"
yield self.st, func, "f()", 2
@@ -953,9 +956,11 @@
yield (self.st, "x=(lambda: (-0.0, 0.0), lambda: (0.0, -0.0))[1]()",
'repr(x)', '(0.0, -0.0)')
+class TestCompilerRevDB(BaseTestCompiler):
+spaceconfig = {"translation.reverse_debugger": True}
+
def test_revdb_metavar(self):
from pypy.interpreter.reverse_debugging import dbstate, setup_revdb
-self.space.config.translation.reverse_debugger = True
self.space.reverse_debugging = True
try:
setup_revdb(self.space)
diff --git a/pypy/interpreter/pyparser/parser.py
b/pypy/interpreter/pyparser/parser.py
--- a/pypy/interpreter/pyparser/parser.py
+++ b/pypy/interpreter/pyparser/parser.py
@@ -28,6 +28,7 @@
new.symbol_ids = self.symbol_ids
new.symbols_names = self.symbol_names
new.keyword_ids = self.keyword_ids
+new.token_to_error_string = self.token_to_error_string
new.dfas = self.dfas
new.labels = self.labels
new.token_ids = self.token_ids
diff --git a/pypy/interpreter/pyparser/pygram.py
b/pypy/interpreter/pyparser/pygram.py
--- a/pypy/interpreter/pyparser/pygram.py
+++ b/pypy/interpreter/pyparser/pygram.py
@@ -23,6 +23,17 @@
python_grammar_no_print.keyword_ids =
python_grammar_no_print.keyword_ids.copy()
del python_grammar_no_print.keyword_ids["print"]
+python_grammar_revdb = python_grammar.shared_copy()
+python_grammar_no_print_revdb = python_grammar_no_print.shared_copy()
+copied_token_ids = python_grammar.token_ids.copy()
+python_grammar_revdb.token_ids = copied_token_ids
+python_grammar_no_print_revdb.token_ids = copied_token_ids
+
+metavar_token_id = pytoken.python_tokens['REVDBMETAVAR']
+# the following line affects python_grammar_no_print too, since they share the
+# dict
+del python_grammar.token_ids[metavar_token_id]
+
class _Tokens(object):
pass
for tok_name, idx in pytoken.python_tokens.iteritems():
@@ -39,3 +50,16 @@
syms._rev_lookup = rev_lookup # for debugging
del _get_python_grammar, _Tokens, tok_name, sym_name, idx
+
+def choose_grammar(print_function, revdb):
+if print_function:
+if revdb:
+return python_grammar_no_print_revdb
+else:
+return python_grammar_no_print
+else:
+if revdb:
+return python_grammar_revdb
+else:
+return python_grammar
+
diff --git a/pypy/interpreter/pyparser/pyparse.py
b/pypy/interpreter/pyparser/pyparse.py
--- a/pypy/interpreter/pyparser/pyparse.py
+++ b/pypy/interpreter/pyparser/pyparse.py
@@ -165,10 +165,9 @@
compile_info.last_future_import = last_future_import
compile_info.flags |= newflags
-if compile_info.flags & consts.CO_FUTURE_PRINT_FUNCTION:
-self.grammar = pygram.python_grammar_no_print
-else:
-self.grammar = pygram.python_grammar
+self.grammar = pygram.choose_grammar(
+print_function=compile_info.flags &
consts.CO_FUTURE_PRINT_FUNCTION,
+revdb=self.space.config.translation.reverse_debugger)
try:
for token in tokens:
diff --git a/pypy/interpreter/pyparser/test/test_pyparse.py
b/pypy/interpreter/pyparser/test/test_pyparse.py
--- a/pypy/interpreter/pyparser/test/test_pyparse.py
+++ b/pypy/interpreter/pyparser/test/test_pyparse.py
@@ -168,13 +168,11 @@
assert expected_tree == tree
def test_revdb_dollar_num(self):
-self.parse('$0')
-self.parse('$5')
-self.parse('$42')
-self.parse('2+$42.attrname')
-py.test.raises(SyntaxError, self.parse, '$')
-py.test.raises(SyntaxError, self.parse, '$a')
-py.test.raises(SyntaxError, self.parse, '$.5')
+assert not self.space.config.translation.reverse_debugger
+py.test.raises(SyntaxError, self.parse, '$0')
+
[pypy-commit] pypy default: merge pyparser-improvements-3
Author: Carl Friedrich Bolz-Tereick
Branch:
Changeset: r94730:e85e93d7927e
Date: 2018-06-06 15:11 +0200
http://bitbucket.org/pypy/pypy/changeset/e85e93d7927e/
Log:merge pyparser-improvements-3
some small refactorings in interpreter/pyparser and module/parser
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -27,3 +27,8 @@
The reverse-debugger branch has been merged. For more information, see
https://bitbucket.org/pypy/revdb
+
+
+.. branch: pyparser-improvements-3
+
+Small refactorings in the Python parser.
diff --git a/pypy/interpreter/pyparser/future.py
b/pypy/interpreter/pyparser/future.py
--- a/pypy/interpreter/pyparser/future.py
+++ b/pypy/interpreter/pyparser/future.py
@@ -43,7 +43,7 @@
self.tok = self.tokens[index]
def skip(self, n):
-if self.tok[0] == n:
+if self.tok.token_type == n:
self.next()
return True
else:
@@ -51,7 +51,7 @@
def skip_name(self, name):
from pypy.interpreter.pyparser import pygram
-if self.tok[0] == pygram.tokens.NAME and self.tok[1] == name:
+if self.tok.token_type == pygram.tokens.NAME and self.tok.value ==
name:
self.next()
return True
else:
@@ -59,8 +59,8 @@
def next_feature_name(self):
from pypy.interpreter.pyparser import pygram
-if self.tok[0] == pygram.tokens.NAME:
-name = self.tok[1]
+if self.tok.token_type == pygram.tokens.NAME:
+name = self.tok.value
self.next()
if self.skip_name("as"):
self.skip(pygram.tokens.NAME)
@@ -101,7 +101,7 @@
# somewhere inside the last __future__ import statement
# (at the start would be fine too, but it's easier to grab a
# random position inside)
-last_position = (it.tok[2], it.tok[3])
+last_position = (it.tok.lineno, it.tok.column)
result |= future_flags.get_compiler_feature(it.next_feature_name())
while it.skip(pygram.tokens.COMMA):
result |= future_flags.get_compiler_feature(it.next_feature_name())
diff --git a/pypy/interpreter/pyparser/parser.py
b/pypy/interpreter/pyparser/parser.py
--- a/pypy/interpreter/pyparser/parser.py
+++ b/pypy/interpreter/pyparser/parser.py
@@ -34,6 +34,18 @@
new.token_ids = self.token_ids
return new
+
+def classify(self, token):
+"""Find the label for a token."""
+if token.token_type == self.KEYWORD_TOKEN:
+label_index = self.keyword_ids.get(token.value, -1)
+if label_index != -1:
+return label_index
+label_index = self.token_ids.get(token.token_type, -1)
+if label_index == -1:
+raise ParseError("invalid token", token)
+return label_index
+
def _freeze_(self):
# Remove some attributes not used in parsing.
try:
@@ -66,6 +78,33 @@
b[pos] |= bit
return str(b)
+
+class Token(object):
+def __init__(self, token_type, value, lineno, column, line):
+self.token_type = token_type
+self.value = value
+self.lineno = lineno
+# 0-based offset
+self.column = column
+self.line = line
+
+def __repr__(self):
+return "Token(%s, %s)" % (self.token_type, self.value)
+
+def __eq__(self, other):
+# for tests
+return (
+self.token_type == other.token_type and
+self.value == other.value and
+self.lineno == other.lineno and
+self.column == other.column and
+self.line == other.line
+)
+
+def __ne__(self, other):
+return not self == other
+
+
class Node(object):
__slots__ = ("type", )
@@ -106,6 +145,11 @@
self.lineno = lineno
self.column = column
+@staticmethod
+def fromtoken(token):
+return Terminal(
+token.token_type, token.value, token.lineno, token.column)
+
def __repr__(self):
return "Terminal(type=%s, value=%r)" % (self.type, self.value)
@@ -194,20 +238,14 @@
class ParseError(Exception):
-def __init__(self, msg, token_type, value, lineno, column, line,
- expected=-1, expected_str=None):
+def __init__(self, msg, token, expected=-1, expected_str=None):
self.msg = msg
-self.token_type = token_type
-self.value = value
-self.lineno = lineno
-# this is a 0-based index
-self.column = column
-self.line = line
+self.token = token
self.expected = expected
self.expected_str = expected_str
def __str__(self):
-return "ParserError(%s, %r)" % (self.token_type, self.value)
+return "ParserError(%s)" % (self.token, )
class StackEntry(object):
@@ -250,8 +288,8 @@
self.root = None
self.stack
