Revision: 17386
Author: [email protected]
Date: Fri Oct 25 07:51:46 2013 UTC
Log: Experimental scanner: more misc fixes around invalid tokens.
This decreases the failures (21 -> 16) for the js files in the source tree.
BUG=
[email protected]
Review URL: https://codereview.chromium.org/43203003
http://code.google.com/p/v8/source/detail?r=17386
Modified:
/branches/experimental/parser/src/lexer/lexer.re
=======================================
--- /branches/experimental/parser/src/lexer/lexer.re Mon Oct 21 15:30:42
2013 UTC
+++ /branches/experimental/parser/src/lexer/lexer.re Fri Oct 25 07:51:46
2013 UTC
@@ -259,7 +259,7 @@
any = [\000-\377];
whitespace_char = [ \t\v\f\r];
whitespace = whitespace_char+;
- identifier_start_ = [$_a-zA-Z];
+ identifier_start = [$_a-zA-Z];
identifier_char = [$_a-zA-Z0-9];
not_identifier_char = any\identifier_char\[\\];
line_terminator = [\n\r]+;
@@ -347,6 +347,10 @@
<Normal> '0x' hex_digit+ not_identifier_char {
PUSH_TOKEN_LOOKAHEAD(Token::NUMBER); }
<Normal> "." digit+ maybe_exponent not_identifier_char {
PUSH_TOKEN_LOOKAHEAD(Token::NUMBER); }
<Normal> digit+ ("." digit+)? maybe_exponent not_identifier_char {
PUSH_TOKEN_LOOKAHEAD(Token::NUMBER); }
+
+ <Normal> '0x' hex_digit+ "\\u" [0-9a-fA-F]{4} { if
(ValidIdentifierStart()) { YYSETCONDITION(kConditionIdentifier); } else {
YYSETCONDITION(kConditionIdentifierIllegal); } cursor_ -= 6;
send(Token::ILLEGAL); start_ = cursor_; cursor_ += 6; goto yy0; }
+ <Normal> "." digit+ maybe_exponent "\\u" [0-9a-fA-F]{4} { if
(ValidIdentifierStart()) { YYSETCONDITION(kConditionIdentifier); } else {
YYSETCONDITION(kConditionIdentifierIllegal); } cursor_ -= 6;
send(Token::ILLEGAL); start_ = cursor_; cursor_ += 6; goto yy0; }
+ <Normal> digit+ ("." digit+)? maybe_exponent "\\u" [0-9a-fA-F]{4} { if
(ValidIdentifierStart()) { YYSETCONDITION(kConditionIdentifier); } else {
YYSETCONDITION(kConditionIdentifierIllegal); } cursor_ -= 6;
send(Token::ILLEGAL); start_ = cursor_; cursor_ += 6; goto yy0; }
<Normal> "(" { PUSH_TOKEN(Token::LPAREN); }
<Normal> ")" { PUSH_TOKEN(Token::RPAREN); }
@@ -381,12 +385,12 @@
<Normal> ["] :=> DoubleQuoteString
<Normal> ['] :=> SingleQuoteString
- <Normal> identifier_start_ :=> Identifier
- <Normal> "\\u" [0-9a-fA-F]{4} { if (ValidIdentifierStart()) {
YYSETCONDITION(kConditionIdentifier); goto yy0; }
YYSETCONDITION(kConditionIdentifierIllegal); send(Token::ILLEGAL); start_ =
cursor_; goto yy0; }
+ <Normal> identifier_start :=> Identifier
+ <Normal> "\\u" [0-9a-fA-F]{4} { if (ValidIdentifierStart()) {
YYSETCONDITION(kConditionIdentifier); goto yy0; } send(Token::ILLEGAL);
start_ = cursor_; goto yy0; }
<Normal> "\\" { PUSH_TOKEN(Token::ILLEGAL); }
<Normal> eof { PUSH_EOF_AND_RETURN();}
- <Normal> any { PUSH_TOKEN(Token::ILLEGAL); }
+ <Normal> any :=> IdentifierIllegal
<DoubleQuoteString> "\\\\" { goto yy0; }
<DoubleQuoteString> "\\\"" { goto yy0; }
@@ -409,11 +413,13 @@
<SingleQuoteString> any { goto yy0; }
<Identifier> identifier_char+ { goto yy0; }
- <Identifier> "\\u" [0-9a-fA-F]{4} { if (ValidIdentifierPart()) goto
yy0; YYSETCONDITION(kConditionIdentifierIllegal); send(Token::ILLEGAL); }
+ <Identifier> "\\u" [0-9a-fA-F]{4} { if (ValidIdentifierPart()) {
YYSETCONDITION(kConditionIdentifier); goto yy0; }
YYSETCONDITION(kConditionNormal); send(Token::ILLEGAL); start_ = cursor_;
goto yy0; }
<Identifier> "\\" { PUSH_TOKEN(Token::ILLEGAL); }
<Identifier> any {
PUSH_TOKEN_LOOKAHEAD(Token::IDENTIFIER); }
- <IdentifierIllegal> identifier_char+ { goto yy0; }
+ <IdentifierIllegal> identifier_start {
PUSH_TOKEN_LOOKAHEAD(Token::ILLEGAL); }
+ <IdentifierIllegal> identifier_char\identifier_start { goto yy0; }
+ <IdentifierIllegal> "\\u" [0-9a-fA-F]{4} { if (ValidIdentifierStart())
{ cursor_ -= 6; PUSH_TOKEN(Token::ILLEGAL); } goto yy0; }
<IdentifierIllegal> "\\"+ { goto yy0; }
<IdentifierIllegal> any {
PUSH_TOKEN_LOOKAHEAD(Token::ILLEGAL); }
--
--
v8-dev mailing list
[email protected]
http://groups.google.com/group/v8-dev
---
You received this message because you are subscribed to the Google Groups "v8-dev" group.
To unsubscribe from this group and stop receiving emails from it, send an email
to [email protected].
For more options, visit https://groups.google.com/groups/opt_out.
