Reviewers: marja,
Message:
Committed patchset #1 manually as r17964.
Description:
Experimental parser: add everything_else class
[email protected]
BUG=
Committed: https://code.google.com/p/v8/source/detail?r=17964
Please review this at https://codereview.chromium.org/80513003/
SVN Base: https://v8.googlecode.com/svn/branches/experimental/parser
Affected files (+54, -30 lines):
M tools/lexer_generator/code_generator.jinja
M tools/lexer_generator/code_generator.py
M tools/lexer_generator/transition_keys.py
Index: tools/lexer_generator/code_generator.jinja
diff --git a/tools/lexer_generator/code_generator.jinja
b/tools/lexer_generator/code_generator.jinja
index
f44ac647618e7bba2fb47ae38af0e8ec11eaaacc..c590244ec37e713c0fc9184f58023cf8aae4b9e1
100644
--- a/tools/lexer_generator/code_generator.jinja
+++ b/tools/lexer_generator/code_generator.jinja
@@ -18,23 +18,36 @@
(yych == 0 && cursor_ >= buffer_end_)
{%- elif r[1] == 'zero' -%}
(yych == 0 && cursor_ < buffer_end_)
- {%- elif r[1] == 'byte_order_mark' and encoding == 'utf16'-%}
- (yych == 0xfffe || yych == 0xfeff)
- {%- elif r[1] == 'non_latin_1_whitespace' and encoding == 'utf16'-%}
- {# FIXME: Add and use unicode_cache_->InNonAsciiWhitespace #}
- (yych > 255 && unicode_cache_->IsWhiteSpace(yych))
- {%- elif r[1] == 'non_latin_1_letter' and encoding == 'utf16'-%}
- {# FIXME: Add and use unicode_cache_->InNonAsciiLetter #}
- (yych > 255 && unicode_cache_->IsLetter(yych))
- {%- elif r[1] == 'non_latin1_identifier_part_not_letter' and
encoding == 'utf16'-%}
- (yych > 255 && unicode_cache_->IsIdentifierPartNotLetter(yych))
- {%- elif r[1] == 'non_latin1_line_terminator' and encoding
== 'utf16'-%}
- (yych > 255 && unicode_cache_->IsLineTerminator(yych))
- {%- else -%}
+ {%- elif encoding == 'latin1' -%}
false /* {{r[1]}} */
+ {%- elif encoding == 'utf16' -%}
+ {%- if r[1] == 'byte_order_mark' -%}
+ (yych == 0xfffe || yych == 0xfeff)
+ {%- elif r[1] == 'non_latin_1_whitespace' -%}
+ {# FIXME: Add and use unicode_cache_->InNonAsciiWhitespace #}
+ (yych > 255 && unicode_cache_->IsWhiteSpace(yych))
+ {%- elif r[1] == 'non_latin_1_letter' -%}
+ {# FIXME: Add and use unicode_cache_->InNonAsciiLetter #}
+ (yych > 255 && unicode_cache_->IsLetter(yych))
+ {%- elif r[1] == 'non_latin_1_identifier_part_not_letter' -%}
+ (yych > 255 && unicode_cache_->IsIdentifierPartNotLetter(yych))
+ {%- elif r[1] == 'non_latin_1_line_terminator' -%}
+ (yych > 255 && unicode_cache_->IsLineTerminator(yych))
+ {%- elif r[1] == 'non_latin_1_everything_else' -%}
+ {# FIXME: Optimize this away #}
+ (yych > 255 &&
+ !unicode_cache_->IsWhiteSpace(yych) &&
+ !unicode_cache_->IsLetter(yych) &&
+ !unicode_cache_->IsIdentifierPartNotLetter(yych) &&
+ !unicode_cache_->IsLineTerminator(yych))
+ {%- else %}
+ uncompilable code for {{encoding}} {{r[0]}} {{r[1]}}
+ {%- endif -%}
+ {%- else -%}
+ uncompilable code for {{encoding}} {{r[0]}} {{r[1]}}
{%- endif -%}
{%- else -%}
- false
+ uncompilable code for {{encoding}} {{r[0]}} {{r[1]}}
{%- endif -%}
{%- endfor -%}
{%- endmacro -%}
@@ -126,6 +139,12 @@
}
{% endfor -%}
+ {%- for key, transition_state_id in state['deferred_transitions'] %}
+ if ({{do_key(key)}}) { // deferred transition
+ {{ do_transition(transition_state_id) }}
+ }
+ {% endfor -%}
+
{%- set match_action = state.match_action -%}
{%- if match_action %}
Index: tools/lexer_generator/code_generator.py
diff --git a/tools/lexer_generator/code_generator.py
b/tools/lexer_generator/code_generator.py
index
823815edaa93169461603c66e97a90dba8e8700b..52a45a11c94218464657296bec78afa84c8ef73d
100644
--- a/tools/lexer_generator/code_generator.py
+++ b/tools/lexer_generator/code_generator.py
@@ -130,6 +130,7 @@ class CodeGenerator:
'original_node_number' : state.node_number(),
'transitions' : transitions,
'switch_transitions' : [],
+ 'deferred_transitions' : [],
'disjoint_keys' : disjoint_keys,
'inline' : None,
'depth' : None,
@@ -167,21 +168,24 @@ class CodeGenerator:
state['inline'] = inline
return count + 1 if inline else count
- @staticmethod
- def __split_transitions(split_count, state):
+ def __split_transitions(self, split_count, state):
'''Goes through the transitions for 'state' and decides which of them
should
use the if statement and which should use the switch statement.'''
assert not state['switch_transitions']
(distinct_keys, ranges) = (state['distinct_keys'], state['ranges'])
- if distinct_keys <= 7 or float(distinct_keys)/float(ranges) >= 7.0:
- return split_count
- switch_transitions = []
+ no_switch = distinct_keys <= 7 or float(distinct_keys)/float(ranges)
= 7.0
if_transitions = []
+ switch_transitions = []
+ deferred_transitions = []
for (ranges, node_id) in state['transitions']:
i = []
s = []
+ d = []
for r in ranges:
+ # all class checks will be deferred to after all other checks
if r[0] == 'CLASS':
+ d.append(r)
+ elif no_switch:
i.append(r)
else:
s.append(r[1])
@@ -189,9 +193,12 @@ class CodeGenerator:
if_transitions.append((i, node_id))
if s:
switch_transitions.append((s, node_id))
+ if d:
+ deferred_transitions.append((d, node_id))
state['transitions'] = if_transitions
state['switch_transitions'] = switch_transitions
- return split_count + 1
+ state['deferred_transitions'] = deferred_transitions
+ return split_count + (0 if no_switch else 1)
def __canonicalize_traversal(self):
dfa_states = []
@@ -219,12 +226,9 @@ class CodeGenerator:
elif self.__log:
print "no inlining"
# split transitions
- if self.__switching:
- switched = reduce(CodeGenerator.__split_transitions, dfa_states, 0)
- if self.__log:
- print "%s states use switch (instead of if)" % switched
- elif self.__log:
- print "no switching"
+ switched = reduce(self.__split_transitions, dfa_states, 0)
+ if self.__log:
+ print "%s states use switch (instead of if)" % switched
def process(self):
Index: tools/lexer_generator/transition_keys.py
diff --git a/tools/lexer_generator/transition_keys.py
b/tools/lexer_generator/transition_keys.py
index
d6bd0e8e8baebb02805c8d52b423446d20454f69..d28fac941e80db9cdb8ebb6eb0b1e0cc20b4e153
100644
--- a/tools/lexer_generator/transition_keys.py
+++ b/tools/lexer_generator/transition_keys.py
@@ -42,11 +42,12 @@ class TransitionKey:
# ranges.
'non_latin_1_whitespace' : (256, 256),
'non_latin_1_letter' : (257, 257),
- 'non_latin1_identifier_part_not_letter' : (258, 258),
- 'non_latin1_line_terminator' : (259, 259),
+ 'non_latin_1_identifier_part_not_letter' : (258, 258),
+ 'non_latin_1_line_terminator' : (259, 259),
'eos' : (260, 260),
'zero' : (261, 261),
'byte_order_mark' : (262, 262),
+ 'non_latin_1_everything_else' : (263, 263),
}
__lower_bound = min(__class_bounds.values(), key=lambda item: item[0])[0]
__upper_bound = max(__class_bounds.values(), key=lambda item: item[1])[1]
@@ -65,10 +66,10 @@ class TransitionKey:
__class_bounds['non_latin_1_letter']],
'line_terminator' : [
(10, 10), (13, 13),
- __class_bounds['non_latin1_line_terminator']],
+ __class_bounds['non_latin_1_line_terminator']],
'identifier_part_not_letter' : [
(48, 57), (95, 95),
- __class_bounds['non_latin1_identifier_part_not_letter']],
+ __class_bounds['non_latin_1_identifier_part_not_letter']],
}
@staticmethod
--
--
v8-dev mailing list
[email protected]
http://groups.google.com/group/v8-dev
---
You received this message because you are subscribed to the Google Groups "v8-dev" group.
To unsubscribe from this group and stop receiving emails from it, send an email
to [email protected].
For more options, visit https://groups.google.com/groups/opt_out.
