Reviewers: dcarney,
Message:
Committed patchset #1 manually as r18715 (presubmit successful).
Description:
Experimental parser: add tests for subgraph handling.
[email protected]
BUG=
Committed: https://code.google.com/p/v8/source/detail?r=18715
Please review this at https://codereview.chromium.org/143643005/
SVN Base: https://v8.googlecode.com/svn/branches/experimental/parser
Affected files (+55, -6 lines):
M tools/lexer_generator/automaton.py
M tools/lexer_generator/lexer_test.py
Index: tools/lexer_generator/automaton.py
diff --git a/tools/lexer_generator/automaton.py
b/tools/lexer_generator/automaton.py
index
e10fac14f45489e3cb1714ed7eb8f094775d0bde..a911a1bb0f3ae61158ad847927ba261317265eaa
100644
--- a/tools/lexer_generator/automaton.py
+++ b/tools/lexer_generator/automaton.py
@@ -179,7 +179,7 @@ class Automaton(object):
valid_states = self.epsilon_closure(valid_states)
return len(self.terminal_set().intersection(valid_states)) > 0
- def lex(self, string):
+ def lex(self, string, default_action):
last_position = 0
valid_states = self.start_set()
for pos, c in enumerate(string):
@@ -188,7 +188,9 @@ class Automaton(object):
valid_states = transitions
continue
action = Action.dominant_action(valid_states)
- assert action # must invoke default action here
+ if not action:
+ assert default_action
+ action = default_action
yield (action, last_position, pos)
last_position = pos
# lex next token
@@ -196,7 +198,9 @@ class Automaton(object):
assert valid_states
valid_states = self.epsilon_closure(valid_states)
action = Action.dominant_action(valid_states)
- assert action # must invoke default action here
+ if not action:
+ assert default_action
+ action = default_action
yield (action, last_position, len(string))
def to_dot(self):
Index: tools/lexer_generator/lexer_test.py
diff --git a/tools/lexer_generator/lexer_test.py
b/tools/lexer_generator/lexer_test.py
index
f0b7fbca491bef136e5a298da074de85db464870..3c76270fca768d950e697a710b997275f23b479e
100644
--- a/tools/lexer_generator/lexer_test.py
+++ b/tools/lexer_generator/lexer_test.py
@@ -32,10 +32,13 @@ from rule_parser import RuleProcessor
class LexerTestCase(unittest.TestCase):
def __verify_action_stream(self, rules, string, expected):
- expected = map(lambda (action, s) : (Action(None, (action, None)), s),
expected)
- automata = RuleProcessor.parse(rules, 'latin1').default_automata()
+ expected = map(lambda (action, s) : (Action(None, (action, None)), s),
+ expected)
+ rule_processor = RuleProcessor.parse(rules, 'latin1')
+ automata = rule_processor.default_automata()
for automaton in [automata.nfa(), automata.dfa(),
automata.minimal_dfa()]:
- for i, (action, start, stop) in enumerate(automaton.lex(string)):
+ for i, (action, start, stop) in enumerate(
+ automaton.lex(string, rule_processor.default_action)):
self.assertEquals(expected[i][0], action)
self.assertEquals(expected[i][1], string[start : stop])
@@ -102,3 +105,45 @@ class LexerTestCase(unittest.TestCase):
self.__verify_action_stream(rules, 'ke', [('ID', 'ke')])
self.__verify_action_stream(rules, 'key', [('ID', 'key')])
self.__verify_action_stream(rules, 'keys', [('ID', 'keys')])
+
+ def test_simple_subgraph(self):
+ rules = '''
+ <<default>>
+ /[a-z]/ <|ID|Identifier>
+ " " <|SPACE|>
+ <<Identifier>>
+ /[a-z]/ <|ID|continue>
+ '''
+ self.__verify_action_stream(rules, 'a bc def',
+ [('ID', 'a'), ('SPACE', ' '), ('ID', 'bc'),
+ ('SPACE', ' '), ('ID', 'def')])
+
+ def test_entering_subgraph_without_match_action(self):
+ # Note: there is no match action for entering the subgraph. It means
that
+ # one char identifiers are not accepted.
+ rules = '''
+ <<default>>
+ /[a-z]/ <||Identifier>
+ " " <|SPACE|>
+ default_action <ILLEGAL>
+ <<Identifier>>
+ /[a-z]/ <|ID|continue>
+ '''
+ self.__verify_action_stream(rules, 'bc a def',
+ [('ID', 'bc'), ('SPACE', ' '),
('ILLEGAL', 'a'),
+ ('SPACE', ' '), ('ID', 'def')])
+
+ def test_subgraph_with_noncontinue(self):
+ # In the "Identifier" subgraph, we have rules which don't
have "continue".
+ rules = '''
+ <<default>>
+ /[b-z]/ <|ID|Identifier>
+ " " <|SPACE|>
+ <<Identifier>>
+ /[b-z]/ <|ID|continue>
+ /[a]/ <|INVALID|>
+ '''
+ self.__verify_action_stream(rules, 'bc ba de',
+ [('ID', 'bc'), ('SPACE', ' '),
+ ('INVALID', 'ba'), ('SPACE', ' '),
+ ('ID', 'de')])
--
--
v8-dev mailing list
[email protected]
http://groups.google.com/group/v8-dev
---
You received this message because you are subscribed to the Google Groups "v8-dev" group.
To unsubscribe from this group and stop receiving emails from it, send an email
to [email protected].
For more options, visit https://groups.google.com/groups/opt_out.
