Revision: 17841
Author: [email protected]
Date: Mon Nov 18 15:23:23 2013 UTC
Log: Experimental lexer generator: Fix the rest of the failures.
Now we lex properly (modulo regexps):
- all ASCII files in the source tree
- all ASCII files in test262
BUG=
[email protected]
Review URL: https://codereview.chromium.org/74793009
http://code.google.com/p/v8/source/detail?r=17841
Modified:
/branches/experimental/parser/src/lexer/even-more-experimental-scanner.cc
/branches/experimental/parser/src/lexer/even-more-experimental-scanner.h
/branches/experimental/parser/src/lexer/lexer_py.re
/branches/experimental/parser/tools/lexer_generator/code_generator.jinja
=======================================
---
/branches/experimental/parser/src/lexer/even-more-experimental-scanner.cc
Mon Nov 18 14:24:27 2013 UTC
+++
/branches/experimental/parser/src/lexer/even-more-experimental-scanner.cc
Mon Nov 18 15:23:23 2013 UTC
@@ -72,6 +72,7 @@
buffer_end_ = buffer_ + size;
start_ = buffer_;
cursor_ = buffer_;
+ marker_ = buffer_;
}
=======================================
---
/branches/experimental/parser/src/lexer/even-more-experimental-scanner.h
Mon Nov 18 14:24:27 2013 UTC
+++
/branches/experimental/parser/src/lexer/even-more-experimental-scanner.h
Mon Nov 18 15:23:23 2013 UTC
@@ -62,6 +62,7 @@
YYCTYPE* buffer_end_;
YYCTYPE* start_;
YYCTYPE* cursor_;
+ YYCTYPE* marker_;
bool just_seen_line_terminator_;
YYCTYPE yych;
=======================================
--- /branches/experimental/parser/src/lexer/lexer_py.re Mon Nov 18 12:20:04
2013 UTC
+++ /branches/experimental/parser/src/lexer/lexer_py.re Mon Nov 18 15:23:23
2013 UTC
@@ -65,7 +65,7 @@
"!" <|push_token(NOT)|>
"//" <||SingleLineComment>
-"/*" <||MultiLineComment>
+"/*" <{marker_ = cursor_ - 2;}||MultiLineComment>
"<!--" <||SingleLineComment>
"<!-" <|{
@@ -195,9 +195,9 @@
<<DoubleQuoteString>>
"\\" line_terminator_sequence <||continue>
-/\\[xX][:hex_digit:]{2}/ <||continue>
+/\\[x][:hex_digit:]{2}/ <||continue>
/\\[u][:hex_digit:]{4}/ <||continue>
-/\\[^xXu\r\n]/ <||continue>
+/\\[^xu\r\n]/ <||continue>
"\\" <|push_token(ILLEGAL)|>
/\n|\r/ <|push_token(ILLEGAL)|>
"\"" <|push_token(STRING)|>
@@ -207,9 +207,9 @@
<<SingleQuoteString>>
# TODO subgraph for '\'
"\\" line_terminator_sequence <||continue>
-/\\[xX][:hex_digit:]{2}/ <||continue>
+/\\[x][:hex_digit:]{2}/ <||continue>
/\\[u][:hex_digit:]{4}/ <||continue>
-/\\[^xXu\r\n]/ <||continue>
+/\\[^xu\r\n]/ <||continue>
"\\" <|push_token(ILLEGAL)|>
/\n|\r/ <|push_token(ILLEGAL)|>
"'" <|push_token(STRING)|>
@@ -234,5 +234,5 @@
# TODO find a way to generate the below rule
/\*+[^\/*]/ <||continue>
line_terminator <push_line_terminator||continue>
-eof <|skip_and_terminate|>
+eof <|{start_ = marker_; BACKWARD(); PUSH_TOKEN(Token::ILLEGAL);}|>
catch_all <||continue>
=======================================
---
/branches/experimental/parser/tools/lexer_generator/code_generator.jinja
Mon Nov 18 14:24:27 2013 UTC
+++
/branches/experimental/parser/tools/lexer_generator/code_generator.jinja
Mon Nov 18 15:23:23 2013 UTC
@@ -118,6 +118,10 @@
#define FORWARD() { \
yych = *(++cursor_); \
}
+
+#define BACKWARD() { \
+ yych = *(--cursor_); \
+}
#define SKIP() { \
start_ = cursor_; \
--
--
v8-dev mailing list
[email protected]
http://groups.google.com/group/v8-dev
---
You received this message because you are subscribed to the Google Groups "v8-dev" group.
To unsubscribe from this group and stop receiving emails from it, send an email
to [email protected].
For more options, visit https://groups.google.com/groups/opt_out.
