Revision: 17542
Author: [email protected]
Date: Thu Nov 7 08:25:59 2013 UTC
Log: Experimental lexer generator: transfer actions across epsilon
transitions correctly when constructing the nfa.
This produces a reasonable nfa (with actions) with the following grammar:
<default>
"break" { BREAK } <<break>>
/[a-z]+/ { ID } <<break>>
BUG=
[email protected]
Review URL: https://codereview.chromium.org/62223002
http://code.google.com/p/v8/source/detail?r=17542
Modified:
/branches/experimental/parser/tools/lexer_generator/generator.py
/branches/experimental/parser/tools/lexer_generator/nfa.py
/branches/experimental/parser/tools/lexer_generator/rule_parser.py
=======================================
--- /branches/experimental/parser/tools/lexer_generator/generator.py Wed
Nov 6 15:54:37 2013 UTC
+++ /branches/experimental/parser/tools/lexer_generator/generator.py Thu
Nov 7 08:25:59 2013 UTC
@@ -80,9 +80,8 @@
builder.set_character_classes(parser_state.character_classes)
for k, v in parser_state.rules.items():
graphs = []
- for (graph, code, action) in v['regex']:
- # graphs.append(NfaBuilder.add_action(graph, (action, identifier)))
- graphs.append(graph)
+ for (graph, code, action, precedence) in v['regex']:
+ graphs.append(NfaBuilder.add_action(graph, (code, action,
precedence)))
rule_map[k] = builder.nfa(NfaBuilder.or_graphs(graphs))
html_data = []
for rule_name, nfa in rule_map.items():
=======================================
--- /branches/experimental/parser/tools/lexer_generator/nfa.py Wed Nov 6
15:45:04 2013 UTC
+++ /branches/experimental/parser/tools/lexer_generator/nfa.py Thu Nov 7
08:25:59 2013 UTC
@@ -333,14 +333,24 @@
f = lambda acc, state: acc | state.key_matches(key)
transitions = reduce(f, nfa_state_set, set())
match_states = set()
- actions = set()
+ actions = []
for (state, action) in transitions:
match_states.add(state)
if action:
- actions.add(action)
+ actions.append(action)
+
+ # Pull in actions which can be taken with an epsilon transition
from the
+ # match state.
+ e = TransitionKey.epsilon()
+ if e in state.transitions():
+ for e_trans in state.transitions()[e]:
+ if e_trans[1]:
+ actions.append(e_trans[1])
+
assert len(match_states) == len(transitions)
- assert not actions or len(actions) == 1
- action = iter(actions).next() if actions else None
+
+ actions.sort()
+ action = actions[0] if actions else None
transition_state = Nfa.__to_dfa(match_states, dfa_nodes, end_node)
dfa_nodes[name]['transitions'][key] = (transition_state, action)
return name
=======================================
--- /branches/experimental/parser/tools/lexer_generator/rule_parser.py Wed
Nov 6 15:45:04 2013 UTC
+++ /branches/experimental/parser/tools/lexer_generator/rule_parser.py Thu
Nov 7 08:25:59 2013 UTC
@@ -47,6 +47,7 @@
class RuleParser:
tokens = RuleLexer.tokens
+ __rule_precedence_counter = 0
def __init__(self):
self.__state = None
@@ -95,7 +96,8 @@
| composite_regex_or_default empty action
| composite_regex_or_default code empty'''
rules = self.__state.rules[self.__state.current_state]
- rule = (p[1], p[2], p[3])
+ rule = (p[1], RuleParser.__rule_precedence_counter, p[2], p[3])
+ RuleParser.__rule_precedence_counter += 1
if p[1] == 'default':
assert not rules['default']
rules['default'] = rule
--
--
v8-dev mailing list
[email protected]
http://groups.google.com/group/v8-dev
---
You received this message because you are subscribed to the Google Groups "v8-dev" group.
To unsubscribe from this group and stop receiving emails from it, send an email
to [email protected].
For more options, visit https://groups.google.com/groups/opt_out.
