Reviewers: dcarney,
Message:
Committed patchset #1 manually as r17386 (presubmit successful).
Description:
Experimental scanner: more misc fixes around invalid tokens.
This decreases the failures (21 -> 16) for the js files in the source tree.
BUG=
[email protected]
Committed: https://code.google.com/p/v8/source/detail?r=17386
Please review this at https://codereview.chromium.org/43203003/
SVN Base: https://v8.googlecode.com/svn/branches/experimental/parser
Affected files (+12, -6 lines):
M src/lexer/lexer.re
Index: src/lexer/lexer.re
diff --git a/src/lexer/lexer.re b/src/lexer/lexer.re
index
36a0b3f3480fdb300d714ba4dcf6cc747748e475..78cb9e5aef04173fed6f426de718cd6de6950b22
100644
--- a/src/lexer/lexer.re
+++ b/src/lexer/lexer.re
@@ -259,7 +259,7 @@ start_:
any = [\000-\377];
whitespace_char = [ \t\v\f\r];
whitespace = whitespace_char+;
- identifier_start_ = [$_a-zA-Z];
+ identifier_start = [$_a-zA-Z];
identifier_char = [$_a-zA-Z0-9];
not_identifier_char = any\identifier_char\[\\];
line_terminator = [\n\r]+;
@@ -348,6 +348,10 @@ start_:
<Normal> "." digit+ maybe_exponent not_identifier_char {
PUSH_TOKEN_LOOKAHEAD(Token::NUMBER); }
<Normal> digit+ ("." digit+)? maybe_exponent not_identifier_char {
PUSH_TOKEN_LOOKAHEAD(Token::NUMBER); }
+ <Normal> '0x' hex_digit+ "\\u" [0-9a-fA-F]{4} { if
(ValidIdentifierStart()) { YYSETCONDITION(kConditionIdentifier); } else {
YYSETCONDITION(kConditionIdentifierIllegal); } cursor_ -= 6;
send(Token::ILLEGAL); start_ = cursor_; cursor_ += 6; goto yy0; }
+ <Normal> "." digit+ maybe_exponent "\\u" [0-9a-fA-F]{4} { if
(ValidIdentifierStart()) { YYSETCONDITION(kConditionIdentifier); } else {
YYSETCONDITION(kConditionIdentifierIllegal); } cursor_ -= 6;
send(Token::ILLEGAL); start_ = cursor_; cursor_ += 6; goto yy0; }
+ <Normal> digit+ ("." digit+)? maybe_exponent "\\u" [0-9a-fA-F]{4} { if
(ValidIdentifierStart()) { YYSETCONDITION(kConditionIdentifier); } else {
YYSETCONDITION(kConditionIdentifierIllegal); } cursor_ -= 6;
send(Token::ILLEGAL); start_ = cursor_; cursor_ += 6; goto yy0; }
+
<Normal> "(" { PUSH_TOKEN(Token::LPAREN); }
<Normal> ")" { PUSH_TOKEN(Token::RPAREN); }
<Normal> "[" { PUSH_TOKEN(Token::LBRACK); }
@@ -381,12 +385,12 @@ start_:
<Normal> ["] :=> DoubleQuoteString
<Normal> ['] :=> SingleQuoteString
- <Normal> identifier_start_ :=> Identifier
- <Normal> "\\u" [0-9a-fA-F]{4} { if (ValidIdentifierStart()) {
YYSETCONDITION(kConditionIdentifier); goto yy0; }
YYSETCONDITION(kConditionIdentifierIllegal); send(Token::ILLEGAL); start_ =
cursor_; goto yy0; }
+ <Normal> identifier_start :=> Identifier
+ <Normal> "\\u" [0-9a-fA-F]{4} { if (ValidIdentifierStart()) {
YYSETCONDITION(kConditionIdentifier); goto yy0; } send(Token::ILLEGAL);
start_ = cursor_; goto yy0; }
<Normal> "\\" { PUSH_TOKEN(Token::ILLEGAL); }
<Normal> eof { PUSH_EOF_AND_RETURN();}
- <Normal> any { PUSH_TOKEN(Token::ILLEGAL); }
+ <Normal> any :=> IdentifierIllegal
<DoubleQuoteString> "\\\\" { goto yy0; }
<DoubleQuoteString> "\\\"" { goto yy0; }
@@ -409,11 +413,13 @@ start_:
<SingleQuoteString> any { goto yy0; }
<Identifier> identifier_char+ { goto yy0; }
- <Identifier> "\\u" [0-9a-fA-F]{4} { if (ValidIdentifierPart()) goto
yy0; YYSETCONDITION(kConditionIdentifierIllegal); send(Token::ILLEGAL); }
+ <Identifier> "\\u" [0-9a-fA-F]{4} { if (ValidIdentifierPart()) {
YYSETCONDITION(kConditionIdentifier); goto yy0; }
YYSETCONDITION(kConditionNormal); send(Token::ILLEGAL); start_ = cursor_;
goto yy0; }
<Identifier> "\\" { PUSH_TOKEN(Token::ILLEGAL); }
<Identifier> any {
PUSH_TOKEN_LOOKAHEAD(Token::IDENTIFIER); }
- <IdentifierIllegal> identifier_char+ { goto yy0; }
+ <IdentifierIllegal> identifier_start {
PUSH_TOKEN_LOOKAHEAD(Token::ILLEGAL); }
+ <IdentifierIllegal> identifier_char\identifier_start { goto yy0; }
+ <IdentifierIllegal> "\\u" [0-9a-fA-F]{4} { if (ValidIdentifierStart())
{ cursor_ -= 6; PUSH_TOKEN(Token::ILLEGAL); } goto yy0; }
<IdentifierIllegal> "\\"+ { goto yy0; }
<IdentifierIllegal> any {
PUSH_TOKEN_LOOKAHEAD(Token::ILLEGAL); }
--
--
v8-dev mailing list
[email protected]
http://groups.google.com/group/v8-dev
---
You received this message because you are subscribed to the Google Groups "v8-dev" group.
To unsubscribe from this group and stop receiving emails from it, send an email
to [email protected].
For more options, visit https://groups.google.com/groups/opt_out.
