Revision: 18052
Author: [email protected]
Date: Mon Nov 25 13:28:43 2013 UTC
Log: Experimental parser: enable transition reduction
[email protected]
BUG=
Review URL: https://codereview.chromium.org/85623003
http://code.google.com/p/v8/source/detail?r=18052
Modified:
/branches/experimental/parser/tools/lexer_generator/code_generator.jinja
/branches/experimental/parser/tools/lexer_generator/code_generator.py
/branches/experimental/parser/tools/lexer_generator/dfa_optimizer.py
/branches/experimental/parser/tools/lexer_generator/generator.py
=======================================
---
/branches/experimental/parser/tools/lexer_generator/code_generator.jinja
Mon Nov 25 09:05:44 2013 UTC
+++
/branches/experimental/parser/tools/lexer_generator/code_generator.jinja
Mon Nov 25 13:28:43 2013 UTC
@@ -60,6 +60,12 @@
DO_LINE_TERMINATOR();
{% elif type == 'token' %}
DO_TOKEN(Token::{{value}})
+ {% elif type == 'goto' %}
+ goto code_after_entry_{{value}};
+ {% elif type == 'store_token' %}
+ stored_token = Token::{{value}};
+ {% elif type == 'do_stored_token' %}
+ DO_TOKEN(stored_token)
{% elif type == 'do_token_and_go_forward' %}
DO_TOKEN_AND_GO_FORWARD(Token::{{value}})
{% elif type == 'harmony_token' %}
@@ -68,6 +74,12 @@
} else {
DO_TOKEN(Token::{{value[1][1]}});
}
+ {% elif type == 'store_harmony_token' %}
+ if (harmony_{{value[0]}}_) {
+ stored_token = Token::{{value[1][0]}};
+ } else {
+ stored_token = Token::{{value[1][1]}};
+ }
{% elif type == 'set_marker' %}
marker_ = cursor_ - {{value}};
{% else %}
@@ -129,6 +141,10 @@
{%- if entry_action %}
{{ dispatch_action(entry_action[0], entry_action[1]) }}
{%- endif %}
+
+ {%- if state['has_goto_after_entry'] -%}
+ code_after_entry_{{node_number}}:
+ {%- endif %}
{%- if debug_print %}
fprintf(stderr, "char at hand is %c (%d)\n", primary_char,
primary_char);
@@ -240,6 +256,7 @@
template<>
void ExperimentalScanner<{{char_type}}>::Scan() {
// Setup environment.
+ Token::Value stored_token;
{{char_type}} primary_char;
if (cursor_ >= buffer_end_) primary_char = 0;
else primary_char = *(cursor_);
@@ -264,7 +281,9 @@
goto code_start;
fell_through:
- CHECK(false);
+ ((void(*)())0)(); // crash
+ // force use of stored_token
+ stored_token = Token::ILLEGAL;
}
} }
=======================================
--- /branches/experimental/parser/tools/lexer_generator/code_generator.py
Mon Nov 25 10:36:04 2013 UTC
+++ /branches/experimental/parser/tools/lexer_generator/code_generator.py
Mon Nov 25 13:28:43 2013 UTC
@@ -131,6 +131,7 @@
'deferred_transitions' : [],
'long_char_transitions' : [],
'disjoint_keys' : disjoint_keys,
+ 'has_goto_after_entry' : False,
'inline' : None,
'depth' : None,
'action' : action,
@@ -288,6 +289,26 @@
assert len(dfa_states) == self.__dfa.node_count()
# store states
self.__dfa_states = dfa_states
+ return id_map
+
+ def __rewrite_gotos(self):
+ goto_map = {}
+ for state in self.__dfa_states:
+ if (state['match_action'] and
+ state['match_action'][0] == 'do_stored_token'):
+ assert not state['match_action'][1] in goto_map
+ goto_map[state['match_action'][1]] = state['node_number']
+ state['has_goto_after_entry'] = True
+ for state in self.__dfa_states:
+ if state['match_action'] and state['match_action'][0] == 'goto':
+ state['match_action'] = ('goto',
goto_map[state['match_action'][1]])
+
+ def process(self):
+
+ id_map = self.__canonicalize_traversal()
+ self.__rewrite_gotos()
+
+ dfa_states = self.__dfa_states
# set nodes to inline
if self.__inline:
inlined = reduce(self.__set_inline, dfa_states, 0)
@@ -303,12 +324,6 @@
for state in dfa_states:
self.__rewrite_deferred_transitions(state)
- def process(self):
-
- self.__canonicalize_traversal()
-
- dfa_states = self.__dfa_states
-
default_action = self.__default_action
assert(default_action and default_action.match_action())
default_action = default_action.match_action()
=======================================
--- /branches/experimental/parser/tools/lexer_generator/dfa_optimizer.py
Mon Nov 25 12:18:26 2013 UTC
+++ /branches/experimental/parser/tools/lexer_generator/dfa_optimizer.py
Mon Nov 25 13:28:43 2013 UTC
@@ -149,10 +149,10 @@
assert not state_replacements
self.__dfa.visit_all_states(replace_state)
# now patch up all states with stores
- for state in store_states:
- old_action = states[state]['action']
- match_action = ('do_stored_token', None)
- states[state]['action'] = replacement_action(old_action,
match_action)
+ for state_id in store_states:
+ old_action = states[state_id]['action']
+ match_action = ('do_stored_token', state_id)
+ states[state_id]['action'] = replacement_action(old_action,
match_action)
start_name = name(self.__dfa.start_state())
if self.__log:
print 'gotos inserted %s' % counters['gotos']
=======================================
--- /branches/experimental/parser/tools/lexer_generator/generator.py Mon
Nov 25 12:18:26 2013 UTC
+++ /branches/experimental/parser/tools/lexer_generator/generator.py Mon
Nov 25 13:28:43 2013 UTC
@@ -98,7 +98,7 @@
parser.add_argument('--input')
parser.add_argument('--code')
parser.add_argument('--encoding', default='latin1')
- parser.add_argument('--optimize-default', action='store_true')
+ parser.add_argument('--no-optimize-default', action='store_true')
parser.add_argument('--no-minimize-default', action='store_true')
parser.add_argument('--no-verify-default', action='store_true')
parser.add_argument('--no-inline', action='store_true')
@@ -115,7 +115,7 @@
with open(re_file, 'r') as f:
rule_processor = RuleProcessor.parse(f.read(), args.encoding)
- if args.optimize_default:
+ if not args.no_optimize_default:
rule_processor.default_automata().optimize_dfa(log = args.verbose)
if minimize_default:
--
--
v8-dev mailing list
[email protected]
http://groups.google.com/group/v8-dev
---
You received this message because you are subscribed to the Google Groups "v8-dev" group.
To unsubscribe from this group and stop receiving emails from it, send an email
to [email protected].
For more options, visit https://groups.google.com/groups/opt_out.
