Revision: 18715
Author: [email protected]
Date: Tue Jan 21 15:59:38 2014 UTC
Log: Experimental parser: add tests for subgraph handling.
[email protected]
BUG=
Review URL: https://codereview.chromium.org/143643005
http://code.google.com/p/v8/source/detail?r=18715
Modified:
/branches/experimental/parser/tools/lexer_generator/automaton.py
/branches/experimental/parser/tools/lexer_generator/lexer_test.py
=======================================
--- /branches/experimental/parser/tools/lexer_generator/automaton.py Mon
Nov 25 15:42:28 2013 UTC
+++ /branches/experimental/parser/tools/lexer_generator/automaton.py Tue
Jan 21 15:59:38 2014 UTC
@@ -179,7 +179,7 @@
valid_states = self.epsilon_closure(valid_states)
return len(self.terminal_set().intersection(valid_states)) > 0
- def lex(self, string):
+ def lex(self, string, default_action):
last_position = 0
valid_states = self.start_set()
for pos, c in enumerate(string):
@@ -188,7 +188,9 @@
valid_states = transitions
continue
action = Action.dominant_action(valid_states)
- assert action # must invoke default action here
+ if not action:
+ assert default_action
+ action = default_action
yield (action, last_position, pos)
last_position = pos
# lex next token
@@ -196,7 +198,9 @@
assert valid_states
valid_states = self.epsilon_closure(valid_states)
action = Action.dominant_action(valid_states)
- assert action # must invoke default action here
+ if not action:
+ assert default_action
+ action = default_action
yield (action, last_position, len(string))
def to_dot(self):
=======================================
--- /branches/experimental/parser/tools/lexer_generator/lexer_test.py Fri
Nov 22 08:40:59 2013 UTC
+++ /branches/experimental/parser/tools/lexer_generator/lexer_test.py Tue
Jan 21 15:59:38 2014 UTC
@@ -32,10 +32,13 @@
class LexerTestCase(unittest.TestCase):
def __verify_action_stream(self, rules, string, expected):
- expected = map(lambda (action, s) : (Action(None, (action, None)), s),
expected)
- automata = RuleProcessor.parse(rules, 'latin1').default_automata()
+ expected = map(lambda (action, s) : (Action(None, (action, None)), s),
+ expected)
+ rule_processor = RuleProcessor.parse(rules, 'latin1')
+ automata = rule_processor.default_automata()
for automaton in [automata.nfa(), automata.dfa(),
automata.minimal_dfa()]:
- for i, (action, start, stop) in enumerate(automaton.lex(string)):
+ for i, (action, start, stop) in enumerate(
+ automaton.lex(string, rule_processor.default_action)):
self.assertEquals(expected[i][0], action)
self.assertEquals(expected[i][1], string[start : stop])
@@ -102,3 +105,45 @@
self.__verify_action_stream(rules, 'ke', [('ID', 'ke')])
self.__verify_action_stream(rules, 'key', [('ID', 'key')])
self.__verify_action_stream(rules, 'keys', [('ID', 'keys')])
+
+ def test_simple_subgraph(self):
+ rules = '''
+ <<default>>
+ /[a-z]/ <|ID|Identifier>
+ " " <|SPACE|>
+ <<Identifier>>
+ /[a-z]/ <|ID|continue>
+ '''
+ self.__verify_action_stream(rules, 'a bc def',
+ [('ID', 'a'), ('SPACE', ' '), ('ID', 'bc'),
+ ('SPACE', ' '), ('ID', 'def')])
+
+ def test_entering_subgraph_without_match_action(self):
+ # Note: there is no match action for entering the subgraph. It means
that
+ # one char identifiers are not accepted.
+ rules = '''
+ <<default>>
+ /[a-z]/ <||Identifier>
+ " " <|SPACE|>
+ default_action <ILLEGAL>
+ <<Identifier>>
+ /[a-z]/ <|ID|continue>
+ '''
+ self.__verify_action_stream(rules, 'bc a def',
+ [('ID', 'bc'), ('SPACE', ' '),
('ILLEGAL', 'a'),
+ ('SPACE', ' '), ('ID', 'def')])
+
+ def test_subgraph_with_noncontinue(self):
+ # In the "Identifier" subgraph, we have rules which don't
have "continue".
+ rules = '''
+ <<default>>
+ /[b-z]/ <|ID|Identifier>
+ " " <|SPACE|>
+ <<Identifier>>
+ /[b-z]/ <|ID|continue>
+ /[a]/ <|INVALID|>
+ '''
+ self.__verify_action_stream(rules, 'bc ba de',
+ [('ID', 'bc'), ('SPACE', ' '),
+ ('INVALID', 'ba'), ('SPACE', ' '),
+ ('ID', 'de')])
--
--
v8-dev mailing list
[email protected]
http://groups.google.com/group/v8-dev
---
You received this message because you are subscribed to the Google Groups "v8-dev" group.
To unsubscribe from this group and stop receiving emails from it, send an email
to [email protected].
For more options, visit https://groups.google.com/groups/opt_out.
