Revision: 18672
Author: [email protected]
Date: Fri Jan 17 16:36:05 2014 UTC
Log: Experimental lexer: track one-byteness on the fly.
BUG=
[email protected]
Review URL: https://codereview.chromium.org/136793025
http://code.google.com/p/v8/source/detail?r=18672
Modified:
/branches/experimental/parser/src/lexer/experimental-scanner.h
/branches/experimental/parser/tools/lexer_generator/code_generator.jinja
=======================================
--- /branches/experimental/parser/src/lexer/experimental-scanner.h Wed Jan
15 08:23:00 2014 UTC
+++ /branches/experimental/parser/src/lexer/experimental-scanner.h Fri Jan
17 16:36:05 2014 UTC
@@ -238,6 +238,7 @@
int beg_pos;
int end_pos;
bool has_escapes;
+ bool is_onebyte;
};
struct LiteralDesc {
=======================================
---
/branches/experimental/parser/tools/lexer_generator/code_generator.jinja
Wed Jan 15 09:31:26 2014 UTC
+++
/branches/experimental/parser/tools/lexer_generator/code_generator.jinja
Fri Jan 17 16:36:05 2014 UTC
@@ -105,6 +105,9 @@
{{dispatch_match_action('backtrack', ('1', 'ILLEGAL'))}}
{% elif type == 'skip' %}
RESET_START();
+ {%- if encoding == 'utf16'-%}
+ next_.is_onebyte = true;
+ {%- endif -%}
goto state_entry_0;
{% elif type == 'skip_and_terminate' %}
RESET_START();
@@ -112,6 +115,9 @@
{{dispatch_match_action('terminate', None)}}
{% elif type == 'line_terminator' %}
RESET_START();
+ {%- if encoding == 'utf16'-%}
+ next_.is_onebyte = true;
+ {%- endif -%}
has_line_terminator_before_next_ = true;
goto state_entry_0;
{% elif type == 'token' %}
@@ -264,6 +270,7 @@
{%- if state['long_char_transitions'] -%}
if ({{long_char_check()}}) {
+ next_.is_onebyte = false;
{{long_char_create()}}
{%- for key, transition_state_id in state['long_char_transitions'] %}
if ({{do_key(key)}}) { // long_char transition
@@ -323,6 +330,7 @@
// Setup environment.
next_.has_escapes = false;
+ next_.is_onebyte = true;
Token::Value stored_token;
const {{char_type}} * marker;
{{char_type}} primary_char;
--
--
v8-dev mailing list
[email protected]
http://groups.google.com/group/v8-dev
---
You received this message because you are subscribed to the Google Groups "v8-dev" group.
To unsubscribe from this group and stop receiving emails from it, send an email
to [email protected].
For more options, visit https://groups.google.com/groups/opt_out.
