Revision: 17287
Author: [email protected]
Date: Mon Oct 21 11:29:05 2013 UTC
Log: Experimental parser: parsing \u stuff correctly.
Now it tokenizes jquery the same way as the baseline.
BUG=
[email protected]
Review URL: https://codereview.chromium.org/31863003
http://code.google.com/p/v8/source/detail?r=17287
Modified:
/branches/experimental/parser/src/lexer/lexer.re
=======================================
--- /branches/experimental/parser/src/lexer/lexer.re Mon Oct 21 11:10:55
2013 UTC
+++ /branches/experimental/parser/src/lexer/lexer.re Mon Oct 21 11:29:05
2013 UTC
@@ -44,6 +44,7 @@
kConditionDoubleQuoteString,
kConditionSingleQuoteString,
kConditionIdentifier,
+ kConditionIdentifierIllegal,
kConditionSingleLineComment,
kConditionMultiLineComment,
kConditionHtmlComment
@@ -197,8 +198,7 @@
whitespace = whitespace_char+;
identifier_start_ = [$_a-zA-Z];
identifier_char = [$_a-zA-Z0-9];
- not_identifier_char = any\identifier_char;
- illegal_after_identifier = [\\];
+ not_identifier_char = any\identifier_char\[\\];
line_terminator = [\n\r]+;
digit = [0-9];
hex_digit = [0-9a-fA-F];
@@ -318,6 +318,9 @@
<Normal> ['] :=> SingleQuoteString
<Normal> identifier_start_ :=> Identifier
+ <Normal> "\\u0000" :=> IdentifierIllegal
+ <Normal> "\\u" [0-9a-fA-F]{4} :=> Identifier
+ <Normal> "\\" { PUSH_TOKEN(Token::ILLEGAL); }
<Normal> eof { PUSH_EOF_AND_RETURN();}
<Normal> any { PUSH_TOKEN(Token::ILLEGAL); }
@@ -341,8 +344,14 @@
<SingleQuoteString> any { goto yy0; }
<Identifier> identifier_char+ { goto yy0; }
- <Identifier> illegal_after_identifier { PUSH_TOKEN(Token::ILLEGAL); }
+ <Identifier> "\\u0000" :=> IdentifierIllegal
+ <Identifier> "\\u" [0-9a-fA-F]{4} { goto yy0; }
+ <Identifier> "\\" { PUSH_TOKEN(Token::ILLEGAL); }
<Identifier> any {
PUSH_TOKEN_LOOKAHEAD(Token::IDENTIFIER); }
+
+ <IdentifierIllegal> identifier_char+ { goto yy0; }
+ <IdentifierIllegal> "\\"+ { goto yy0; }
+ <IdentifierIllegal> any {
PUSH_TOKEN_LOOKAHEAD(Token::ILLEGAL); }
<SingleLineComment> line_terminator { PUSH_LINE_TERMINATOR();}
<SingleLineComment> eof { PUSH_LINE_TERMINATOR();}
--
--
v8-dev mailing list
[email protected]
http://groups.google.com/group/v8-dev
---
You received this message because you are subscribed to the Google Groups "v8-dev" group.
To unsubscribe from this group and stop receiving emails from it, send an email
to [email protected].
For more options, visit https://groups.google.com/groups/opt_out.
