[v8-dev] Experimental scanner: More fixes. (issue 43993002)

[marja]

Reviewers: dcarney,

Message:
Committed patchset #1 manually as r17395 (presubmit successful).
Description:
Experimental scanner: More fixes.

Failures now down to 9 (from 16).

"marker_" was unused; using it for the purpose of going back to a defined  
point
in the character stream. E.g., if we're parsing "\illegalstuff\22222" and  
notice
a "\" after the number. At this point we need to push 22222 as ILLEGAL, so  
we
need to know where it started.

TBR=dcar...@chromium.org
BUG=

Committed: https://code.google.com/p/v8/source/detail?r=17395

Please review this at https://codereview.chromium.org/43993002/

SVN Base: https://v8.googlecode.com/svn/branches/experimental/parser

Affected files (+10, -11 lines):
  M src/lexer/lexer.re


Index: src/lexer/lexer.re
diff --git a/src/lexer/lexer.re b/src/lexer/lexer.re
index  
78cb9e5aef04173fed6f426de718cd6de6950b22..d51771910fe1cb56526b581e3379a55906c86854  
100644
--- a/src/lexer/lexer.re
+++ b/src/lexer/lexer.re
@@ -266,6 +266,7 @@ start_:
     digit = [0-9];
     hex_digit = [0-9a-fA-F];
     maybe_exponent = ('e' [-+]? digit+)?;
+    number = ('0x' hex_digit+) | ("." digit+ maybe_exponent) | (digit+  
("." digit+)? maybe_exponent);

     <Normal> "break" not_identifier_char      {  
PUSH_TOKEN_LOOKAHEAD(Token::BREAK); }
     <Normal> "case" not_identifier_char       {  
PUSH_TOKEN_LOOKAHEAD(Token::CASE); }
@@ -344,13 +345,9 @@ start_:
     <Normal> "<"           { PUSH_TOKEN(Token::LT); }
     <Normal> ">"           { PUSH_TOKEN(Token::GT); }

-    <Normal> '0x' hex_digit+ not_identifier_char                     {  
PUSH_TOKEN_LOOKAHEAD(Token::NUMBER); }
-    <Normal> "." digit+ maybe_exponent not_identifier_char           {  
PUSH_TOKEN_LOOKAHEAD(Token::NUMBER); }
-    <Normal> digit+ ("." digit+)? maybe_exponent not_identifier_char {  
PUSH_TOKEN_LOOKAHEAD(Token::NUMBER); }
-
-    <Normal> '0x' hex_digit+ "\\u" [0-9a-fA-F]{4} { if  
(ValidIdentifierStart()) { YYSETCONDITION(kConditionIdentifier); } else {  
YYSETCONDITION(kConditionIdentifierIllegal); } cursor_ -= 6;  
send(Token::ILLEGAL); start_ = cursor_; cursor_ += 6; goto yy0; }
-    <Normal> "." digit+ maybe_exponent "\\u" [0-9a-fA-F]{4} { if  
(ValidIdentifierStart()) { YYSETCONDITION(kConditionIdentifier); } else {  
YYSETCONDITION(kConditionIdentifierIllegal); } cursor_ -= 6;  
send(Token::ILLEGAL); start_ = cursor_; cursor_ += 6; goto yy0; }
-    <Normal> digit+ ("." digit+)? maybe_exponent "\\u" [0-9a-fA-F]{4} { if  
(ValidIdentifierStart()) { YYSETCONDITION(kConditionIdentifier); } else {  
YYSETCONDITION(kConditionIdentifierIllegal); } cursor_ -= 6;  
send(Token::ILLEGAL); start_ = cursor_; cursor_ += 6; goto yy0; }
+    <Normal> number not_identifier_char {  
PUSH_TOKEN_LOOKAHEAD(Token::NUMBER); }
+    <Normal> number "\\u" [0-9a-fA-F]{4} { cursor_ -= 6;  
send(Token::ILLEGAL); start_ = cursor_; cursor_ += 6; if  
(ValidIdentifierStart()) { YYSETCONDITION(kConditionIdentifier); goto yy0;  
} else { PUSH_TOKEN(Token::ILLEGAL); } }
+    <Normal> number any   { PUSH_TOKEN_LOOKAHEAD(Token::ILLEGAL); }

     <Normal> "("           { PUSH_TOKEN(Token::LPAREN); }
     <Normal> ")"           { PUSH_TOKEN(Token::RPAREN); }
@@ -390,7 +387,7 @@ start_:
     <Normal> "\\"                 { PUSH_TOKEN(Token::ILLEGAL); }

     <Normal> eof           { PUSH_EOF_AND_RETURN();}
-    <Normal> any           :=> IdentifierIllegal
+    <Normal> any           { marker_ = cursor_;  
YYSETCONDITION(kConditionIdentifierIllegal); goto yy0; }

     <DoubleQuoteString> "\\\\"  { goto yy0; }
     <DoubleQuoteString> "\\\""  { goto yy0; }
@@ -418,9 +415,11 @@ start_:
     <Identifier> any               {  
PUSH_TOKEN_LOOKAHEAD(Token::IDENTIFIER); }

     <IdentifierIllegal> identifier_start  {  
PUSH_TOKEN_LOOKAHEAD(Token::ILLEGAL); }
-    <IdentifierIllegal> identifier_char\identifier_start  { goto yy0; }
-    <IdentifierIllegal> "\\u" [0-9a-fA-F]{4} { if (ValidIdentifierStart())  
{ cursor_ -= 6; PUSH_TOKEN(Token::ILLEGAL); } goto yy0; }
-    <IdentifierIllegal> "\\"+             { goto yy0; }
+    <IdentifierIllegal> identifier_char\identifier_start  { marker_ =  
cursor_; goto yy0; }
+    <IdentifierIllegal> "\\u" [0-9a-fA-F]{4} { if (ValidIdentifierStart())  
{ cursor_ -= 6; PUSH_TOKEN(Token::ILLEGAL); } marker_ = cursor_;  
PUSH_TOKEN(Token::ILLEGAL); }
+    <IdentifierIllegal> "\\"+             { marker_ = cursor_; goto yy0; }
+    <IdentifierIllegal> number not_identifier_char { YYCTYPE* temp =  
cursor_; cursor_ = marker_; send(Token::ILLEGAL); cursor_ = temp;  
YYSETCONDITION(kConditionNormal); PUSH_TOKEN_LOOKAHEAD(Token::NUMBER); }
+    <IdentifierIllegal> number "\\u" [0-9a-fA-F]{4} { YYCTYPE* temp =  
cursor_; cursor_ = marker_; send(Token::ILLEGAL); cursor_ = temp;  
send(Token::ILLEGAL); start_ = cursor_; goto yy0; }
     <IdentifierIllegal> any               {  
PUSH_TOKEN_LOOKAHEAD(Token::ILLEGAL); }

     <SingleLineComment> line_terminator { PUSH_LINE_TERMINATOR();}


--
--
v8-dev mailing list
v8-dev@googlegroups.com
http://groups.google.com/group/v8-dev
--- 
You received this message because you are subscribed to the Google Groups "v8-dev" group.
To unsubscribe from this group and stop receiving emails from it, send an email 
to v8-dev+unsubscr...@googlegroups.com.
For more options, visit https://groups.google.com/groups/opt_out.