Revision: 17589
Author: [email protected]
Date: Fri Nov 8 11:52:04 2013 UTC
Log: Experimental parser: better generation
[email protected]
BUG=
Review URL: https://codereview.chromium.org/66533002
http://code.google.com/p/v8/source/detail?r=17589
Modified:
/branches/experimental/parser/src/lexer/lexer_py.re
/branches/experimental/parser/tools/lexer_generator/generator.py
/branches/experimental/parser/tools/lexer_generator/nfa.py
=======================================
--- /branches/experimental/parser/src/lexer/lexer_py.re Wed Nov 6 15:45:04
2013 UTC
+++ /branches/experimental/parser/src/lexer/lexer_py.re Fri Nov 8 11:52:04
2013 UTC
@@ -101,16 +101,63 @@
line_terminator+ { PUSH_LINE_TERMINATOR(); }
whitespace <<continue>>
-"\"" <<DoubleQuoteString>> # TODO mark these transitions as
ignoring this character
+"\"" <<DoubleQuoteString>>
"'" <<SingleQuoteString>>
-identifier_start <<Identifier>> # TODO merge identifier dfa...
+# all keywords
+"break" { PUSH_TOKEN(BREAK); } <<break>>
+"case" { PUSH_TOKEN(CASE); } <<break>>
+"catch" { PUSH_TOKEN(CATCH); } <<break>>
+"class" { PUSH_TOKEN(FUTURE_RESERVED_WORD); } <<break>>
+"const" { PUSH_TOKEN(CONST); } <<break>>
+"continue" { PUSH_TOKEN(CONTINUE); } <<break>>
+"debugger" { PUSH_TOKEN(DEBUGGER); } <<break>>
+"default" { PUSH_TOKEN(DEFAULT); } <<break>>
+"delete" { PUSH_TOKEN(DELETE); } <<break>>
+"do" { PUSH_TOKEN(DO); } <<break>>
+"else" { PUSH_TOKEN(ELSE); } <<break>>
+"enum" { PUSH_TOKEN(FUTURE_RESERVED_WORD); } <<break>>
+"export" { PUSH_TOKEN(FUTURE_RESERVED_WORD); } <<break>>
+"extends" { PUSH_TOKEN(FUTURE_RESERVED_WORD); } <<break>>
+"false" { PUSH_TOKEN(FALSE_LITERAL); } <<break>>
+"finally" { PUSH_TOKEN(FINALLY); } <<break>>
+"for" { PUSH_TOKEN(FOR); } <<break>>
+"function" { PUSH_TOKEN(FUNCTION); } <<break>>
+"if" { PUSH_TOKEN(IF); } <<break>>
+"implements" { PUSH_TOKEN(FUTURE_STRICT_RESERVED_WORD); } <<break>>
+"import" { PUSH_TOKEN(FUTURE_RESERVED_WORD); } <<break>>
+"in" { PUSH_TOKEN(IN); } <<break>>
+"instanceof" { PUSH_TOKEN(INSTANCEOF); } <<break>>
+"interface" { PUSH_TOKEN(FUTURE_STRICT_RESERVED_WORD); } <<break>>
+"let" { PUSH_TOKEN(FUTURE_STRICT_RESERVED_WORD); } <<break>>
+"new" { PUSH_TOKEN(NEW); } <<break>>
+"null" { PUSH_TOKEN(NULL_LITERAL); } <<break>>
+"package" { PUSH_TOKEN(FUTURE_STRICT_RESERVED_WORD); } <<break>>
+"private" { PUSH_TOKEN(FUTURE_STRICT_RESERVED_WORD); } <<break>>
+"protected" { PUSH_TOKEN(FUTURE_STRICT_RESERVED_WORD); } <<break>>
+"public" { PUSH_TOKEN(FUTURE_STRICT_RESERVED_WORD); } <<break>>
+"return" { PUSH_TOKEN(RETURN); } <<break>>
+"static" { PUSH_TOKEN(FUTURE_STRICT_RESERVED_WORD); } <<break>>
+"super" { PUSH_TOKEN(FUTURE_RESERVED_WORD); } <<break>>
+"switch" { PUSH_TOKEN(SWITCH); } <<break>>
+"this" { PUSH_TOKEN(THIS); } <<break>>
+"throw" { PUSH_TOKEN(THROW); } <<break>>
+"true" { PUSH_TOKEN(TRUE_LITERAL); } <<break>>
+"try" { PUSH_TOKEN(TRY); } <<break>>
+"typeof" { PUSH_TOKEN(TYPEOF); } <<break>>
+"var" { PUSH_TOKEN(VAR); } <<break>>
+"void" { PUSH_TOKEN(VOID); } <<break>>
+"while" { PUSH_TOKEN(WHILE); } <<break>>
+"with" { PUSH_TOKEN(WITH); } <<break>>
+"yield" { PUSH_TOKEN(YIELD); } <<break>>
+
+identifier_start <<Identifier>>
/\\u[0-9a-fA-F]{4}/ {
if (V8_LIKELY(ValidIdentifierStart())) {
JUMP(Identifier);
}
PUSH_TOKEN(ILLEGAL);
-}
+} <<Identifier>>
eof <<terminate>>
default { PUSH_TOKEN(ILLEGAL); }
@@ -121,7 +168,7 @@
"\"" { PUSH_TOKEN(STRING); } <<break>>
/\\\n\r?/ <<continue>>
/\\\r\n?/ <<continue>>
-/\n\r/ { PUSH_TOKEN(ILLEGAL); } <<break>>
+/\n|\r/ { PUSH_TOKEN(ILLEGAL); } <<break>>
eof <<terminate_illegal>>
default <<continue>>
@@ -131,7 +178,7 @@
"'" { PUSH_TOKEN(STRING); } <<break>>
/\\\n\r?/ <<continue>>
/\\\r\n?/ <<continue>>
-/\n\r/ { PUSH_TOKEN(ILLEGAL); } <<break>>
+/\n|\r/ { PUSH_TOKEN(ILLEGAL); } <<break>>
eof <<terminate_illegal>>
default <<continue>>
@@ -161,51 +208,3 @@
line_terminator+ { PUSH_LINE_TERMINATOR(); }
eof <<terminate>>
default <<continue>>
-
-<default>
-# all keywords
-"break" { PUSH_TOKEN(BREAK); } <<break>>
-"case" { PUSH_TOKEN(CASE); } <<break>>
-"catch" { PUSH_TOKEN(CATCH); } <<break>>
-"class" { PUSH_TOKEN(FUTURE_RESERVED_WORD); } <<break>>
-"const" { PUSH_TOKEN(CONST); } <<break>>
-"continue" { PUSH_TOKEN(CONTINUE); } <<break>>
-"debugger" { PUSH_TOKEN(DEBUGGER); } <<break>>
-"default" { PUSH_TOKEN(DEFAULT); } <<break>>
-"delete" { PUSH_TOKEN(DELETE); } <<break>>
-"do" { PUSH_TOKEN(DO); } <<break>>
-"else" { PUSH_TOKEN(ELSE); } <<break>>
-"enum" { PUSH_TOKEN(FUTURE_RESERVED_WORD); } <<break>>
-"export" { PUSH_TOKEN(FUTURE_RESERVED_WORD); } <<break>>
-"extends" { PUSH_TOKEN(FUTURE_RESERVED_WORD); } <<break>>
-"false" { PUSH_TOKEN(FALSE_LITERAL); } <<break>>
-"finally" { PUSH_TOKEN(FINALLY); } <<break>>
-"for" { PUSH_TOKEN(FOR); } <<break>>
-"function" { PUSH_TOKEN(FUNCTION); } <<break>>
-"if" { PUSH_TOKEN(IF); } <<break>>
-"implements" { PUSH_TOKEN(FUTURE_STRICT_RESERVED_WORD); } <<break>>
-"import" { PUSH_TOKEN(FUTURE_RESERVED_WORD); } <<break>>
-"in" { PUSH_TOKEN(IN); } <<break>>
-"instanceof" { PUSH_TOKEN(INSTANCEOF); } <<break>>
-"interface" { PUSH_TOKEN(FUTURE_STRICT_RESERVED_WORD); } <<break>>
-"let" { PUSH_TOKEN(FUTURE_STRICT_RESERVED_WORD); } <<break>>
-"new" { PUSH_TOKEN(NEW); } <<break>>
-"null" { PUSH_TOKEN(NULL_LITERAL); } <<break>>
-"package" { PUSH_TOKEN(FUTURE_STRICT_RESERVED_WORD); } <<break>>
-"private" { PUSH_TOKEN(FUTURE_STRICT_RESERVED_WORD); } <<break>>
-"protected" { PUSH_TOKEN(FUTURE_STRICT_RESERVED_WORD); } <<break>>
-"public" { PUSH_TOKEN(FUTURE_STRICT_RESERVED_WORD); } <<break>>
-"return" { PUSH_TOKEN(RETURN); } <<break>>
-"static" { PUSH_TOKEN(FUTURE_STRICT_RESERVED_WORD); } <<break>>
-"super" { PUSH_TOKEN(FUTURE_RESERVED_WORD); } <<break>>
-"switch" { PUSH_TOKEN(SWITCH); } <<break>>
-"this" { PUSH_TOKEN(THIS); } <<break>>
-"throw" { PUSH_TOKEN(THROW); } <<break>>
-"true" { PUSH_TOKEN(TRUE_LITERAL); } <<break>>
-"try" { PUSH_TOKEN(TRY); } <<break>>
-"typeof" { PUSH_TOKEN(TYPEOF); } <<break>>
-"var" { PUSH_TOKEN(VAR); } <<break>>
-"void" { PUSH_TOKEN(VOID); } <<break>>
-"while" { PUSH_TOKEN(WHILE); } <<break>>
-"with" { PUSH_TOKEN(WITH); } <<break>>
-"yield" { PUSH_TOKEN(YIELD); } <<break>>
=======================================
--- /branches/experimental/parser/tools/lexer_generator/generator.py Fri
Nov 8 10:33:44 2013 UTC
+++ /branches/experimental/parser/tools/lexer_generator/generator.py Fri
Nov 8 11:52:04 2013 UTC
@@ -90,10 +90,12 @@
if transition == 'continue':
if not v['default'][1][2] == 'continue':
graph = NfaBuilder.add_continue(graph)
+ else:
+ pass # TODO null key
elif (transition == 'break' or
transition == 'terminate' or
transition == 'terminate_illegal'):
- NfaBuilder.add_action(graph, (-1, transition, None))
+ graph = NfaBuilder.add_action(graph, (10000, transition, None))
else:
assert k == 'default'
graph = NfaBuilder.join_subgraph(graph, transition,
rule_map[transition])
@@ -104,7 +106,7 @@
assert transition == 'continue' or transition == 'break'
if transition == 'continue':
assert k != 'default'
- # graph = NfaBuilder.apply_modifier('*', graph)
+ graph = NfaBuilder.add_incoming_action(graph, (10000, k, None))
if code:
graph = NfaBuilder.add_incoming_action(graph, (precedence, code,
None))
rule_map[k] = graph
=======================================
--- /branches/experimental/parser/tools/lexer_generator/nfa.py Fri Nov 8
10:28:35 2013 UTC
+++ /branches/experimental/parser/tools/lexer_generator/nfa.py Fri Nov 8
11:52:04 2013 UTC
@@ -214,8 +214,11 @@
new_start.close(start)
start = new_start
else:
+ new_end = self.__new_state()
for end in ends:
end.set_transition_action(action)
+ self.__patch_ends(ends, new_end)
+ ends = [new_end]
return (start, ends)
def __continue(self, graph):
--
--
v8-dev mailing list
[email protected]
http://groups.google.com/group/v8-dev
---
You received this message because you are subscribed to the Google Groups "v8-dev" group.
To unsubscribe from this group and stop receiving emails from it, send an email
to [email protected].
For more options, visit https://groups.google.com/groups/opt_out.
