Revision: 17744
Author: [email protected]
Date: Thu Nov 14 12:43:42 2013 UTC
Log: Experimental parser: implement skip
[email protected]
BUG=
Review URL: https://codereview.chromium.org/64913011
http://code.google.com/p/v8/source/detail?r=17744
Modified:
/branches/experimental/parser/src/lexer/even-more-experimental-scanner.cc
/branches/experimental/parser/src/lexer/lexer.re
/branches/experimental/parser/src/lexer/lexer_py.re
/branches/experimental/parser/tools/lexer_generator/code_generator.py
/branches/experimental/parser/tools/lexer_generator/rule_parser.py
=======================================
---
/branches/experimental/parser/src/lexer/even-more-experimental-scanner.cc
Thu Nov 14 12:14:08 2013 UTC
+++
/branches/experimental/parser/src/lexer/even-more-experimental-scanner.cc
Thu Nov 14 12:43:42 2013 UTC
@@ -58,18 +58,6 @@
using namespace v8::internal;
-namespace {
-
-inline int HexValue(uc32 c) {
- c -= '0';
- if (static_cast<unsigned>(c) <= 9) return c;
- c = (c | 0x20) - ('a' - '0'); // detect 0x11..0x16 and 0x31..0x36.
- if (static_cast<unsigned>(c) <= 5) return c + 10;
- return -1;
-}
-
-}
-
namespace v8 {
namespace internal {
=======================================
--- /branches/experimental/parser/src/lexer/lexer.re Tue Nov 12 15:51:17
2013 UTC
+++ /branches/experimental/parser/src/lexer/lexer.re Thu Nov 14 12:43:42
2013 UTC
@@ -77,18 +77,6 @@
using namespace v8::internal;
-namespace {
-
-inline int HexValue(uc32 c) {
- c -= '0';
- if (static_cast<unsigned>(c) <= 9) return c;
- c = (c | 0x20) - ('a' - '0'); // detect 0x11..0x16 and 0x31..0x36.
- if (static_cast<unsigned>(c) <= 5) return c + 10;
- return -1;
-}
-
-}
-
#define PUSH_TOKEN(T) { send(T); SKIP(); }
#define PUSH_TOKEN_LOOKAHEAD(T) { --cursor_; send(T); SKIP(); }
#define PUSH_EOF_AND_RETURN() { send(Token::EOS); eof_ = true; return 1;}
=======================================
--- /branches/experimental/parser/src/lexer/lexer_py.re Thu Nov 14 12:14:08
2013 UTC
+++ /branches/experimental/parser/src/lexer/lexer_py.re Thu Nov 14 12:43:42
2013 UTC
@@ -100,7 +100,7 @@
"," push_token(COMMA)
line_terminator+ { PUSH_LINE_TERMINATOR(); }
-whitespace { SKIP(); } # TODO implement skip
+whitespace <<skip>>
"\"" <<DoubleQuoteString>>
"'" <<SingleQuoteString>>
@@ -159,7 +159,7 @@
}
} <<Identifier>>
-eof { PUSH_TOKEN(Token::EOS); return 0; }
+eof <<terminate>>
default_action push_token(ILLEGAL)
<DoubleQuoteString>
@@ -181,32 +181,26 @@
catch_all <<continue>>
<Identifier>
-identifier_char <<continue>>
+identifier_char push_token(IDENTIFIER) <<continue>>
/\\u[0-9a-fA-F]{4}/ {
if (V8_UNLIKELY(!ValidIdentifierStart())) {
PUSH_TOKEN(Token::ILLEGAL);
}
} <<continue>>
-default_action push_token(IDENTIFIER)
<SingleLineComment>
line_terminator { PUSH_LINE_TERMINATOR(); }
-eof <<terminate>>
catch_all <<continue>>
<MultiLineComment>
-"*/" { SKIP(); goto code_start;}
+"*/" <<skip>>
/\*[^\057]/ <<continue>>
-# need to force action
-line_terminator+ { PUSH_LINE_TERMINATOR(); } <<continue>>
-eof <<terminate>>
+line_terminator { PUSH_LINE_TERMINATOR(); } <<continue>>
catch_all <<continue>>
<HtmlComment>
-"-->" { SKIP(); }
+"-->" <<skip>>
/--./ <<continue>>
/-./ <<continue>>
-# need to force action
-line_terminator+ { PUSH_LINE_TERMINATOR(); } <<continue>>
-eof <<terminate>>
+line_terminator { PUSH_LINE_TERMINATOR(); } <<continue>>
catch_all <<continue>>
=======================================
--- /branches/experimental/parser/tools/lexer_generator/code_generator.py
Thu Nov 14 12:14:08 2013 UTC
+++ /branches/experimental/parser/tools/lexer_generator/code_generator.py
Thu Nov 14 12:43:42 2013 UTC
@@ -75,6 +75,9 @@
elif action.type() == 'terminate_illegal':
code += 'PUSH_TOKEN(Token::ILLEGAL); return 1;'
return code
+ elif action.type() == 'skip':
+ code += 'SKIP(); goto code_start;'
+ return code
code += '''
//fprintf(stderr, "char at hand is %c (%d)\\n", yych, yych);\n'''
=======================================
--- /branches/experimental/parser/tools/lexer_generator/rule_parser.py Thu
Nov 14 11:48:38 2013 UTC
+++ /branches/experimental/parser/tools/lexer_generator/rule_parser.py Thu
Nov 14 12:43:42 2013 UTC
@@ -52,7 +52,7 @@
tokens = RuleLexer.tokens
__rule_precedence_counter = 0
__keyword_transitions = set([
- 'continue', 'break', 'terminate', 'terminate_illegal'])
+ 'continue', 'break', 'terminate', 'terminate_illegal', 'skip'])
def __init__(self):
self.__state = None
@@ -111,6 +111,7 @@
rules = self.__state.rules[self.__state.current_state]
code = p[2]
if p[1] == 'default_action':
+ assert self.__state.current_state == 'default'
assert not rules['default_action']
rules['default_action'] = code
elif p[1] == 'catch_all':
@@ -288,7 +289,8 @@
continues += 1
graph = NfaBuilder.add_continue(graph)
elif (transition == 'terminate' or
- transition == 'terminate_illegal'):
+ transition == 'terminate_illegal' or
+ transition == 'skip'):
assert not code
graph = NfaBuilder.add_action(graph, Action(transition, None,
-1))
else:
--
--
v8-dev mailing list
[email protected]
http://groups.google.com/group/v8-dev
---
You received this message because you are subscribed to the Google Groups "v8-dev" group.
To unsubscribe from this group and stop receiving emails from it, send an email
to [email protected].
For more options, visit https://groups.google.com/groups/opt_out.
