Reviewers: marja,
Message:
Committed patchset #2 manually as r19167 (presubmit successful).
Description:
Experimental parser: better rule tree visualization
[email protected]
BUG=
Committed: https://code.google.com/p/v8/source/detail?r=19167
Please review this at https://codereview.chromium.org/145723010/
SVN Base: https://v8.googlecode.com/svn/branches/experimental/parser
Affected files (+40, -13 lines):
M tools/lexer_generator/nfa_builder.py
M tools/lexer_generator/regex_parser.py
Index: tools/lexer_generator/nfa_builder.py
diff --git a/tools/lexer_generator/nfa_builder.py
b/tools/lexer_generator/nfa_builder.py
index
36a9fc3a3192b7d8831d47e5407c169c11d92800..8877690e78bc34e62c9d7899efcb0d418777a59f
100644
--- a/tools/lexer_generator/nfa_builder.py
+++ b/tools/lexer_generator/nfa_builder.py
@@ -59,10 +59,11 @@ class NfaBuilder(object):
self.__global_end_node = self.__new_state()
return self.__global_end_node
- def __or(self, left, right):
+ def __or(self, *trees):
start = self.__new_state()
ends = []
- for (sub_start, sub_end) in [self.__process(left),
self.__process(right)]:
+ for tree in trees:
+ (sub_start, sub_end) = self.__process(tree)
start.add_epsilon_transition(sub_start)
ends += sub_end
start.close(None)
@@ -110,17 +111,28 @@ class NfaBuilder(object):
midpoints.append(midpoint)
return (start, ends + midpoints)
- def __cat(self, left_tree, right_tree):
- (left, right) = (self.__process(left_tree), self.__process(right_tree))
- self.__patch_ends(left[1], right[0])
- return (left[0], right[1])
+ def __cat(self, *trees):
+ (start, ends) = (None, None)
+ for tree in trees:
+ (sub_start, sub_ends) = self.__process(tree)
+ if start == None:
+ start = sub_start
+ else:
+ assert sub_ends, "this creates unreachable nodes"
+ self.__patch_ends(ends, sub_start)
+ ends = sub_ends
+ return (start, ends)
def __key_state(self, key):
state = self.__new_state()
state.add_unclosed_transition(key)
return (state, [state])
- def __literal(self, char):
+ def __literal(self, chars):
+ terms = map(lambda c : Term('SINGLE_CHAR', c), chars)
+ return self.__process(self.cat_terms(terms))
+
+ def __single_char(self, char):
return self.__key_state(
TransitionKey.single_char(self.__encoding, char))
@@ -285,11 +297,13 @@ class NfaBuilder(object):
@staticmethod
def or_terms(terms):
- return reduce(lambda acc, g: Term('OR', acc, g), terms)
+ if len(terms) == 1: return terms[0]
+ return Term('OR', *terms) if terms else Term.empty()
@staticmethod
def cat_terms(terms):
- return reduce(lambda acc, g: Term('CAT', acc, g), terms)
+ if len(terms) == 1: return terms[0]
+ return Term('CAT', *terms) if terms else Term.empty()
__modifer_map = {
'+': 'ONE_OR_MORE',
Index: tools/lexer_generator/regex_parser.py
diff --git a/tools/lexer_generator/regex_parser.py
b/tools/lexer_generator/regex_parser.py
index
d19a5432570508bef88dd5e8e99fb28ec4fe7ea8..b624584d2d849b91e1ea6dce645e87a50febf7fa
100644
--- a/tools/lexer_generator/regex_parser.py
+++ b/tools/lexer_generator/regex_parser.py
@@ -171,7 +171,7 @@ class RegexParser:
p[0] = self.__cat(p[1], p[2])
def p_fragment(self, p):
- '''fragment : literal maybe_modifier
+ '''fragment : literal_array maybe_modifier
| class maybe_modifier
| group maybe_modifier
| any maybe_modifier
@@ -202,9 +202,22 @@ class RegexParser:
else:
p[0] = ("REPEAT", p[2], p[4])
- def p_literal(self, p):
- '''literal : LITERAL'''
- p[0] = Term('LITERAL', p[1])
+ def p_literal_array(self, p):
+ '''literal_array : literals'''
+ p[0] = Term('LITERAL', ''.join(reversed(p[1])))
+
+ def p_literals(self, p):
+ '''literals : LITERAL maybe_literals'''
+ if not p[2]:
+ p[0] = [p[1]]
+ else:
+ p[2].append(p[1])
+ p[0] = p[2]
+
+ def p_maybe_literals(self, p):
+ '''maybe_literals : literals
+ | empty'''
+ p[0] = p[1]
def p_any(self, p):
'''any : ANY'''
--
--
v8-dev mailing list
[email protected]
http://groups.google.com/group/v8-dev
---
You received this message because you are subscribed to the Google Groups "v8-dev" group.
To unsubscribe from this group and stop receiving emails from it, send an email
to [email protected].
For more options, visit https://groups.google.com/groups/opt_out.
