Reviewers: marja,
Message:
Committed patchset #1 manually as r18796 (presubmit successful).
Description:
Experimental parser: revert read_cursor
[email protected]
BUG=
Committed: https://code.google.com/p/v8/source/detail?r=18796
Please review this at https://codereview.chromium.org/139853014/
SVN Base: https://v8.googlecode.com/svn/branches/experimental/parser
Affected files (+36, -29 lines):
M tools/lexer_generator/code_generator.jinja
M tools/lexer_generator/code_generator.py
Index: tools/lexer_generator/code_generator.jinja
diff --git a/tools/lexer_generator/code_generator.jinja
b/tools/lexer_generator/code_generator.jinja
index
32fb65069d11c345f1e570231ebc5c5d034db394..a4207826cf5ae71c7f2812146705217a8e1ac179
100644
--- a/tools/lexer_generator/code_generator.jinja
+++ b/tools/lexer_generator/code_generator.jinja
@@ -34,8 +34,10 @@
{%- for r in key -%}
{%- if not loop.first %} || {% endif -%}
{%- if r[0] == 'PRIMARY_RANGE' -%}
- {%- if r[1][0] == r[1][1] -%}
+ {%- if r[1][0] == r[1][1] and r[1][0] != 0 -%}
primary_char == {{r[1][0]}}
+ {%- elif r[1][0] == r[1][1] -%}
+ (primary_char == 0 && cursor_ < buffer_end_)
{%- elif r[1][0] == 0 -%}
primary_char <= {{r[1][1]}}
{%- elif r[1][1] == upper_bound and not encoding == 'utf16'-%}
@@ -229,22 +231,14 @@
{% endif -%}
{%- macro do_transition(jump_id) -%}
+ {%- set transition_state_id = jump_table[jump_id][0] -%}
+ {%- set inline_transition = jump_table[jump_id][1] == 'inline' %}
FORWARD();
- if (cursor_ < buffer_end_) {
- {%- set transition_state_id = jump_table[jump_id][0] -%}
- {%- set inline_transition = jump_table[jump_id][1] == 'inline' %}
- {%- if inline_transition %}
- {{ do_dfa_state(transition_state_id) }}
- {% else %}
- {{ jump(jump_id) }}
- {% endif %}
- }
- {% if 'eos' in state['unique_transitions'] -%}
- {{ jump(state['unique_transitions']['eos']) }} // eos handler
- {%- else -%}
- BACKWARD(1);
- {{ jump(state['jump_before_match']) }} // no eos handler
- {%- endif %}
+ {%- if inline_transition %}
+ {{ do_dfa_state(transition_state_id) }}
+ {% else %}
+ {{ jump(jump_id) }}
+ {% endif %}
{%- endmacro -%}
{%- if state['switch_transitions'] -%}
@@ -266,7 +260,13 @@
}
{% endfor -%}
- {%- if state['deferred_transitions'] -%}
+ {%- if 'eos' in state['unique_transitions'] %}
+ if (primary_char == 0 && cursor_ >= buffer_end_) { // eos handler
+ {{ jump(state['unique_transitions']['eos']) }}
+ }
+ {%- endif -%}
+
+ {%- if state['deferred_transitions'] %}
if ({{long_char_check()}}) {
next_.is_onebyte = false;
{{long_char_create()}}
@@ -276,9 +276,7 @@
}
{% endfor -%}
}
- {%- endif-%}
-
- {{ write_label('before_match', node_number) }}
+ {%- endif -%}
{%- set match_action = state.match_action -%}
{%- if match_action %}
@@ -312,8 +310,8 @@
}
#define READ_CURSOR() { \
- ASSERT(cursor_ < buffer_end_); \
- primary_char = *(cursor_); \
+ if (cursor_ >= buffer_end_) primary_char = 0; \
+ else primary_char = *(cursor_); \
}
#ifdef DEBUG
Index: tools/lexer_generator/code_generator.py
diff --git a/tools/lexer_generator/code_generator.py
b/tools/lexer_generator/code_generator.py
index
d70b9b7ddcf51c0269e1ed0e490b1cd1a5479e51..556b07cd7bd1721858d9e25f7857e50d3c156a8b
100644
--- a/tools/lexer_generator/code_generator.py
+++ b/tools/lexer_generator/code_generator.py
@@ -54,7 +54,7 @@ class CodeGenerator:
self.__switching = switching
self.__jump_table = []
- __jump_labels = ['state_entry', 'after_entry_code', 'before_match']
+ __jump_labels = ['state_entry', 'after_entry_code']
@staticmethod
def __transform_state(encoding, state):
@@ -74,6 +74,7 @@ class CodeGenerator:
old_transitions = transitions
transitions = []
(class_keys, distinct_keys, ranges) = (0, 0, 0)
+ zero_transition = None
for key, transition_id in old_transitions:
keys = []
for (t, r) in key.range_iter(encoding):
@@ -83,6 +84,14 @@ class CodeGenerator:
elif t == 'PRIMARY_RANGE':
distinct_keys += r[1] - r[0] + 1
ranges += 1
+ # split 0 out of range, don't bother updating stats
+ assert r[0] >= 0
+ if r[0] == 0:
+ assert zero_transition == None
+ zero_transition = transition_id
+ if r[0] == r[1]:
+ continue
+ r = (r[0] + 1, r[1])
keys.append((t, r))
elif t == 'UNIQUE':
assert r == 'no_match' or r == 'eos'
@@ -92,6 +101,9 @@ class CodeGenerator:
raise Exception()
if keys:
transitions.append((keys, transition_id))
+ # delay zero transition until last
+ if zero_transition != None:
+ transitions.append(([('PRIMARY_RANGE', (0, 0))], transition_id))
return {
'node_number' : None,
'original_node_number' : state.node_number(),
@@ -99,7 +111,7 @@ class CodeGenerator:
# flags for code generator
'can_elide_read' : len(transitions) == 0,
'is_eos_handler' : False,
- 'inline' : None,
+ 'inline' : False,
'must_not_inline' : False,
# transitions for code generator
'if_transitions' : [],
@@ -125,7 +137,6 @@ class CodeGenerator:
return len(self.__jump_table) - 1
def __set_inline(self, count, state):
- assert state['inline'] == None
inline = False
if state['must_not_inline']:
inline = False
@@ -160,7 +171,8 @@ class CodeGenerator:
# all class checks will be deferred to after all other checks
if r[0] == 'CLASS':
d.append(r)
- elif no_switch:
+ # zero must assigned to an if check because of eos check
+ elif no_switch or (r[1][0] == 0):
i.append(r)
else:
s.append(r[1])
@@ -354,9 +366,6 @@ class CodeGenerator:
eos_state_id = state['unique_transitions']['eos']
jump = self.__register_jump(eos_state_id, 'state_entry')
state['unique_transitions']['eos'] = jump
- elif state['transitions']:
- jump = self.__register_jump(state_id, 'before_match')
- state['jump_before_match'] = jump
# now rewrite all nodes created
nodes_created = len(inline_mapping) - len(inline_mapping_in)
assert len(self.__dfa_states) == (
--
--
v8-dev mailing list
[email protected]
http://groups.google.com/group/v8-dev
---
You received this message because you are subscribed to the Google Groups "v8-dev" group.
To unsubscribe from this group and stop receiving emails from it, send an email
to [email protected].
For more options, visit https://groups.google.com/groups/opt_out.
