Reviewers: ,
Message:
Committed patchset #1 manually as r18095 (presubmit successful).
Description:
Experimental scanner: keep track of which strings / identifiers contain an
escape.
BUG=
[email protected]
Committed: https://code.google.com/p/v8/source/detail?r=18095
Please review this at https://codereview.chromium.org/91123002/
SVN Base: https://v8.googlecode.com/svn/branches/experimental/parser
Affected files (+14, -8 lines):
M src/lexer/experimental-scanner.h
M src/lexer/lexer_py.re
M tools/lexer_generator/code_generator.jinja
Index: src/lexer/experimental-scanner.h
diff --git a/src/lexer/experimental-scanner.h
b/src/lexer/experimental-scanner.h
index
05423c6b669a70eaa46c7f7f9e4be26aa7c2ed13..ef65ee5cc1e199559dacadaedd8de47df9ea2e48
100644
--- a/src/lexer/experimental-scanner.h
+++ b/src/lexer/experimental-scanner.h
@@ -188,8 +188,9 @@ class ScannerBase {
// we need to decide if pos is counted in characters or in bytes.
virtual void SeekForward(int pos) = 0;
- // Scans the input as a regular expression pattern, previous
- // character(s) must be /(=). Returns true if a pattern is scanned.
+ // Scans the input as a regular expression pattern, previous
character(s) must
+ // be /(=). Returns true if a pattern is scanned. FIXME: this won't work
for
+ // utf-8 newlines.
virtual bool ScanRegExpPattern(bool seen_equal) = 0;
// Returns true if regexp flags are scanned (always since flags can
// be empty).
@@ -200,6 +201,7 @@ class ScannerBase {
Token::Value token;
int beg_pos;
int end_pos;
+ bool has_escapes;
};
virtual void Scan() = 0;
Index: src/lexer/lexer_py.re
diff --git a/src/lexer/lexer_py.re b/src/lexer/lexer_py.re
index
49c7cb5f0804371765d99ea0bb82aebd7de42807..fce1f68b5456a8762d274cbce9efa9568d70b2a9
100644
--- a/src/lexer/lexer_py.re
+++ b/src/lexer/lexer_py.re
@@ -196,9 +196,9 @@ default_action <do_token_and_go_forward(ILLEGAL)>
<<DoubleQuoteString>>
"\\" line_terminator_sequence <||continue>
-/\\[x][:hex_digit:]{2}/ <||continue>
-/\\[u][:hex_digit:]{4}/ <||continue>
-/\\[^xu:line_terminator:]/ <||continue>
+/\\[x][:hex_digit:]{2}/ <set_has_escapes||continue>
+/\\[u][:hex_digit:]{4}/ <set_has_escapes||continue>
+/\\[^xu:line_terminator:]/ <set_has_escapes||continue>
"\\" <|token(ILLEGAL)|>
line_terminator <|token(ILLEGAL)|>
"\"" <|token(STRING)|>
@@ -208,9 +208,9 @@ catch_all <||continue>
<<SingleQuoteString>>
# TODO subgraph for '\'
"\\" line_terminator_sequence <||continue>
-/\\[x][:hex_digit:]{2}/ <||continue>
-/\\[u][:hex_digit:]{4}/ <||continue>
-/\\[^xu:line_terminator:]/ <||continue>
+/\\[x][:hex_digit:]{2}/ <set_has_escapes||continue>
+/\\[u][:hex_digit:]{4}/ <set_has_escapes||continue>
+/\\[^xu:line_terminator:]/ <set_has_escapes||continue>
"\\" <|token(ILLEGAL)|>
line_terminator <|token(ILLEGAL)|>
"'" <|token(STRING)|>
@@ -223,6 +223,7 @@ identifier_char <|token(IDENTIFIER)|continue>
if (V8_UNLIKELY(!ValidIdentifierPart())) {
goto default_action;
}
+ next_.has_escapes = true;
}|token(IDENTIFIER)|continue>
<<SingleLineComment>>
Index: tools/lexer_generator/code_generator.jinja
diff --git a/tools/lexer_generator/code_generator.jinja
b/tools/lexer_generator/code_generator.jinja
index
6ef81c2b73c78dfd3622b7b73ab0b625a783ee31..8ee95db35be381cccc681d10ce40461f687ab4f3
100644
--- a/tools/lexer_generator/code_generator.jinja
+++ b/tools/lexer_generator/code_generator.jinja
@@ -113,6 +113,8 @@
goto code_after_entry_{{value[3]}};
{% elif type == 'set_marker' %}
marker_ = cursor_ - {{value}};
+ {% elif type == 'set_has_escapes' %}
+ next_.has_escapes = true;
{% else %}
uncompilable code for {{type}}
{% endif -%}
@@ -286,6 +288,7 @@ namespace v8 {
namespace internal {
template<>
void ExperimentalScanner<{{char_type}}>::Scan() {
+ next_.has_escapes = false;
// Setup environment.
Token::Value stored_token;
{{char_type}} primary_char;
--
--
v8-dev mailing list
[email protected]
http://groups.google.com/group/v8-dev
---
You received this message because you are subscribed to the Google Groups "v8-dev" group.
To unsubscribe from this group and stop receiving emails from it, send an email
to [email protected].
For more options, visit https://groups.google.com/groups/opt_out.
