Reviewers: marja,
Message:
Committed patchset #1 manually as r17744.
Description:
Experimental parser: implement skip
[email protected]
BUG=
Committed: https://code.google.com/p/v8/source/detail?r=17744
Please review this at https://codereview.chromium.org/64913011/
SVN Base: https://v8.googlecode.com/svn/branches/experimental/parser
Affected files (+14, -39 lines):
M src/lexer/even-more-experimental-scanner.cc
M src/lexer/lexer.re
M src/lexer/lexer_py.re
M tools/lexer_generator/code_generator.py
M tools/lexer_generator/rule_parser.py
Index: src/lexer/even-more-experimental-scanner.cc
diff --git a/src/lexer/even-more-experimental-scanner.cc
b/src/lexer/even-more-experimental-scanner.cc
index
bcf8c2a0c91c002cbbbb15565997dd3f45006901..805208669d2a1abebc05f85d2f1f559e2d4abfed
100644
--- a/src/lexer/even-more-experimental-scanner.cc
+++ b/src/lexer/even-more-experimental-scanner.cc
@@ -58,18 +58,6 @@
using namespace v8::internal;
-namespace {
-
-inline int HexValue(uc32 c) {
- c -= '0';
- if (static_cast<unsigned>(c) <= 9) return c;
- c = (c | 0x20) - ('a' - '0'); // detect 0x11..0x16 and 0x31..0x36.
- if (static_cast<unsigned>(c) <= 5) return c + 10;
- return -1;
-}
-
-}
-
namespace v8 {
namespace internal {
Index: src/lexer/lexer.re
diff --git a/src/lexer/lexer.re b/src/lexer/lexer.re
index
625ab922b98fa27aa06c92582ce1f146b8994606..42f255523234fb8cd0bed533ea928c15a0eeb24f
100644
--- a/src/lexer/lexer.re
+++ b/src/lexer/lexer.re
@@ -77,18 +77,6 @@ enum Condition {
using namespace v8::internal;
-namespace {
-
-inline int HexValue(uc32 c) {
- c -= '0';
- if (static_cast<unsigned>(c) <= 9) return c;
- c = (c | 0x20) - ('a' - '0'); // detect 0x11..0x16 and 0x31..0x36.
- if (static_cast<unsigned>(c) <= 5) return c + 10;
- return -1;
-}
-
-}
-
#define PUSH_TOKEN(T) { send(T); SKIP(); }
#define PUSH_TOKEN_LOOKAHEAD(T) { --cursor_; send(T); SKIP(); }
#define PUSH_EOF_AND_RETURN() { send(Token::EOS); eof_ = true; return 1;}
Index: src/lexer/lexer_py.re
diff --git a/src/lexer/lexer_py.re b/src/lexer/lexer_py.re
index
cf0f1806edef205a754ac3958df6cf4eb9b9a8b7..4d03a129cdc78c555feff8e2ef757c6e2c64d1c1
100644
--- a/src/lexer/lexer_py.re
+++ b/src/lexer/lexer_py.re
@@ -100,7 +100,7 @@ number push_token(NUMBER)
"," push_token(COMMA)
line_terminator+ { PUSH_LINE_TERMINATOR(); }
-whitespace { SKIP(); } # TODO implement skip
+whitespace <<skip>>
"\"" <<DoubleQuoteString>>
"'" <<SingleQuoteString>>
@@ -159,7 +159,7 @@ identifier_start push_token(IDENTIFIER) <<Identifier>>
}
} <<Identifier>>
-eof { PUSH_TOKEN(Token::EOS); return 0; }
+eof <<terminate>>
default_action push_token(ILLEGAL)
<DoubleQuoteString>
@@ -181,32 +181,26 @@ eof <<terminate_illegal>>
catch_all <<continue>>
<Identifier>
-identifier_char <<continue>>
+identifier_char push_token(IDENTIFIER) <<continue>>
/\\u[0-9a-fA-F]{4}/ {
if (V8_UNLIKELY(!ValidIdentifierStart())) {
PUSH_TOKEN(Token::ILLEGAL);
}
} <<continue>>
-default_action push_token(IDENTIFIER)
<SingleLineComment>
line_terminator { PUSH_LINE_TERMINATOR(); }
-eof <<terminate>>
catch_all <<continue>>
<MultiLineComment>
-"*/" { SKIP(); goto code_start;}
+"*/" <<skip>>
/\*[^\057]/ <<continue>>
-# need to force action
-line_terminator+ { PUSH_LINE_TERMINATOR(); } <<continue>>
-eof <<terminate>>
+line_terminator { PUSH_LINE_TERMINATOR(); } <<continue>>
catch_all <<continue>>
<HtmlComment>
-"-->" { SKIP(); }
+"-->" <<skip>>
/--./ <<continue>>
/-./ <<continue>>
-# need to force action
-line_terminator+ { PUSH_LINE_TERMINATOR(); } <<continue>>
-eof <<terminate>>
+line_terminator { PUSH_LINE_TERMINATOR(); } <<continue>>
catch_all <<continue>>
Index: tools/lexer_generator/code_generator.py
diff --git a/tools/lexer_generator/code_generator.py
b/tools/lexer_generator/code_generator.py
index
979865c2750c6246d083b2cfbe60ace9ac3712f4..eede23327a47128e84a29fdf6fd6d2efdd886316
100644
--- a/tools/lexer_generator/code_generator.py
+++ b/tools/lexer_generator/code_generator.py
@@ -75,6 +75,9 @@ code_%s:
elif action.type() == 'terminate_illegal':
code += 'PUSH_TOKEN(Token::ILLEGAL); return 1;'
return code
+ elif action.type() == 'skip':
+ code += 'SKIP(); goto code_start;'
+ return code
code += '''
//fprintf(stderr, "char at hand is %c (%d)\\n", yych, yych);\n'''
Index: tools/lexer_generator/rule_parser.py
diff --git a/tools/lexer_generator/rule_parser.py
b/tools/lexer_generator/rule_parser.py
index
3875ad2e3ccb553fed9fe472aa56798e30867a3e..d6468a027818490a7db88d076c306b45db22971e
100644
--- a/tools/lexer_generator/rule_parser.py
+++ b/tools/lexer_generator/rule_parser.py
@@ -52,7 +52,7 @@ class RuleParser:
tokens = RuleLexer.tokens
__rule_precedence_counter = 0
__keyword_transitions = set([
- 'continue', 'break', 'terminate', 'terminate_illegal'])
+ 'continue', 'break', 'terminate', 'terminate_illegal', 'skip'])
def __init__(self):
self.__state = None
@@ -111,6 +111,7 @@ class RuleParser:
rules = self.__state.rules[self.__state.current_state]
code = p[2]
if p[1] == 'default_action':
+ assert self.__state.current_state == 'default'
assert not rules['default_action']
rules['default_action'] = code
elif p[1] == 'catch_all':
@@ -288,7 +289,8 @@ class RuleProcessor(object):
continues += 1
graph = NfaBuilder.add_continue(graph)
elif (transition == 'terminate' or
- transition == 'terminate_illegal'):
+ transition == 'terminate_illegal' or
+ transition == 'skip'):
assert not code
graph = NfaBuilder.add_action(graph, Action(transition, None,
-1))
else:
--
--
v8-dev mailing list
[email protected]
http://groups.google.com/group/v8-dev
---
You received this message because you are subscribed to the Google Groups "v8-dev" group.
To unsubscribe from this group and stop receiving emails from it, send an email
to [email protected].
For more options, visit https://groups.google.com/groups/opt_out.
