Reviewers: marja,
Message:
Committed patchset #1 manually as r18773 (presubmit successful).
Description:
Experimental parser: cleanup deferred transitions
[email protected]
BUG=
Committed: https://code.google.com/p/v8/source/detail?r=18773
Please review this at https://codereview.chromium.org/145623002/
SVN Base: https://v8.googlecode.com/svn/branches/experimental/parser
Affected files (+13, -43 lines):
M tools/lexer_generator/code_generator.jinja
M tools/lexer_generator/code_generator.py
Index: tools/lexer_generator/code_generator.jinja
diff --git a/tools/lexer_generator/code_generator.jinja
b/tools/lexer_generator/code_generator.jinja
index
f34f9211bdc5b0b99a6624f5e295ea96deba2bc5..32fb65069d11c345f1e570231ebc5c5d034db394
100644
--- a/tools/lexer_generator/code_generator.jinja
+++ b/tools/lexer_generator/code_generator.jinja
@@ -43,14 +43,6 @@
{%- else -%}
({{r[1][0]}} <= primary_char && primary_char <= {{r[1][1]}})
{%- endif -%}
- {%- elif r[0] == 'CLASS' -%}
- {%- if r[1] == 'eos' -%}
- (primary_char == 0 && cursor_ >= buffer_end_)
- {%- elif r[1] == 'zero' -%}
- (primary_char == 0 && cursor_ < buffer_end_)
- {%- else %}
- uncompilable code for {{encoding}} {{r[0]}} {{r[1]}}
- {%- endif -%}
{# These classes require long_char and to be outside the primary range
#}
{%- elif r[0] == 'LONG_CHAR_CLASS' and encoding in ['utf16', 'utf8']
-%}
{%- if r[1] == 'byte_order_mark' -%}
@@ -274,17 +266,11 @@
}
{% endfor -%}
- {%- for key, jump_id in state['deferred_transitions'] %}
- if ({{do_key(key)}}) { // deferred transition
- {{ do_transition(jump_id) }}
- }
- {% endfor -%}
-
- {%- if state['long_char_transitions'] -%}
+ {%- if state['deferred_transitions'] -%}
if ({{long_char_check()}}) {
next_.is_onebyte = false;
{{long_char_create()}}
- {%- for key, jump_id in state['long_char_transitions'] %}
+ {%- for key, jump_id in state['deferred_transitions'] %}
if ({{do_key(key)}}) { // long_char transition
{{ do_transition(jump_id) }}
}
Index: tools/lexer_generator/code_generator.py
diff --git a/tools/lexer_generator/code_generator.py
b/tools/lexer_generator/code_generator.py
index
00b39e5a80cecb341093e2c1cfa6cdedd505f089..d70b9b7ddcf51c0269e1ed0e490b1cd1a5479e51
100644
--- a/tools/lexer_generator/code_generator.py
+++ b/tools/lexer_generator/code_generator.py
@@ -105,7 +105,6 @@ class CodeGenerator:
'if_transitions' : [],
'switch_transitions' : [],
'deferred_transitions' : [],
- 'long_char_transitions' : [],
'unique_transitions' : unique_transitions,
# state actions
'entry_action' : entry_action,
@@ -184,7 +183,6 @@ class CodeGenerator:
}
def __rewrite_deferred_transitions(self, state):
- assert not state['long_char_transitions']
transitions = state['deferred_transitions']
if not transitions:
return
@@ -192,58 +190,45 @@ class CodeGenerator:
bom = 'byte_order_mark'
catch_all = 'non_primary_everything_else'
all_classes = set(encoding.class_name_iter())
- fast_classes = set([])
- call_classes = all_classes - fast_classes - set([bom, catch_all])
+ call_classes = all_classes - set([bom, catch_all])
def remap_transition(class_name):
if class_name in call_classes:
return ('LONG_CHAR_CLASS', 'call', self.__call_map[class_name])
if class_name == bom:
return ('LONG_CHAR_CLASS', class_name)
raise Exception(class_name)
- fast_transitions = []
long_class_transitions = []
long_class_map = {}
catchall_transition = None
# loop through and remove catch_all_transitions
for (classes, transition_node_id) in transitions:
- ft = []
lct = []
has_catch_all = False
for (class_type, class_name) in classes:
- if class_name in fast_classes:
- ft.append((class_type, class_name))
+ assert not class_name in long_class_map
+ long_class_map[class_name] = transition_node_id
+ if class_name == catch_all:
+ assert not has_catch_all
+ assert catchall_transition == None
+ has_catch_all = True
else:
- assert not class_name in long_class_map
- long_class_map[class_name] = transition_node_id
- if class_name == catch_all:
- assert not has_catch_all
- assert catchall_transition == None
- has_catch_all = True
- else:
- lct.append(remap_transition(class_name))
- if ft:
- fast_transitions.append((ft, transition_node_id))
+ lct.append(remap_transition(class_name))
if has_catch_all:
catchall_transition = (lct, transition_node_id)
elif lct:
long_class_transitions.append((lct, transition_node_id))
- # all transitions are fast
- if not long_class_map:
- return
if catchall_transition:
- catchall_transitions = all_classes - fast_classes
+ catchall_transitions = all_classes
for class_name in long_class_map.iterkeys():
catchall_transitions.remove(class_name)
assert not catchall_transitions, "class inversion not unimplemented"
- # split deferred transitions
- state['deferred_transitions'] = fast_transitions
if catchall_transition:
catchall_transition = [
([('LONG_CHAR_CLASS', 'catch_all')], catchall_transition[1])]
else:
catchall_transition = []
- state['long_char_transitions'] = (long_class_transitions +
- catchall_transition) # must be last
+ state['deferred_transitions'] = (long_class_transitions +
+ catchall_transition) # must be last
@staticmethod
def __reorder(current_node_number, id_map, dfa_states):
@@ -337,7 +322,6 @@ class CodeGenerator:
transition_names = [
'switch_transitions',
'if_transitions',
- 'long_char_transitions',
'deferred_transitions']
end_offset = start_id + count
assert len(self.__dfa_states) == end_offset
--
--
v8-dev mailing list
[email protected]
http://groups.google.com/group/v8-dev
---
You received this message because you are subscribed to the Google Groups "v8-dev" group.
To unsubscribe from this group and stop receiving emails from it, send an email
to [email protected].
For more options, visit https://groups.google.com/groups/opt_out.
