Reviewers: marja,
Message:
Committed patchset #1 manually as r18052 (presubmit successful).
Description:
Experimental parser: enable transition reduction
[email protected]
BUG=
Committed: https://code.google.com/p/v8/source/detail?r=18052
Please review this at https://codereview.chromium.org/85623003/
SVN Base: https://v8.googlecode.com/svn/branches/experimental/parser
Affected files (+47, -13 lines):
M tools/lexer_generator/code_generator.jinja
M tools/lexer_generator/code_generator.py
M tools/lexer_generator/dfa_optimizer.py
M tools/lexer_generator/generator.py
Index: tools/lexer_generator/code_generator.jinja
diff --git a/tools/lexer_generator/code_generator.jinja
b/tools/lexer_generator/code_generator.jinja
index
68cdf8c42cb35740599d192210fcaae2c29eef34..3b5031623b92a902a3646dc96d3779443eabdfd8
100644
--- a/tools/lexer_generator/code_generator.jinja
+++ b/tools/lexer_generator/code_generator.jinja
@@ -60,6 +60,12 @@
DO_LINE_TERMINATOR();
{% elif type == 'token' %}
DO_TOKEN(Token::{{value}})
+ {% elif type == 'goto' %}
+ goto code_after_entry_{{value}};
+ {% elif type == 'store_token' %}
+ stored_token = Token::{{value}};
+ {% elif type == 'do_stored_token' %}
+ DO_TOKEN(stored_token)
{% elif type == 'do_token_and_go_forward' %}
DO_TOKEN_AND_GO_FORWARD(Token::{{value}})
{% elif type == 'harmony_token' %}
@@ -68,6 +74,12 @@
} else {
DO_TOKEN(Token::{{value[1][1]}});
}
+ {% elif type == 'store_harmony_token' %}
+ if (harmony_{{value[0]}}_) {
+ stored_token = Token::{{value[1][0]}};
+ } else {
+ stored_token = Token::{{value[1][1]}};
+ }
{% elif type == 'set_marker' %}
marker_ = cursor_ - {{value}};
{% else %}
@@ -130,6 +142,10 @@
{{ dispatch_action(entry_action[0], entry_action[1]) }}
{%- endif %}
+ {%- if state['has_goto_after_entry'] -%}
+ code_after_entry_{{node_number}}:
+ {%- endif %}
+
{%- if debug_print %}
fprintf(stderr, "char at hand is %c (%d)\n", primary_char,
primary_char);
{% endif -%}
@@ -240,6 +256,7 @@ namespace internal {
template<>
void ExperimentalScanner<{{char_type}}>::Scan() {
// Setup environment.
+ Token::Value stored_token;
{{char_type}} primary_char;
if (cursor_ >= buffer_end_) primary_char = 0;
else primary_char = *(cursor_);
@@ -264,7 +281,9 @@ void ExperimentalScanner<{{char_type}}>::Scan() {
goto code_start;
fell_through:
- CHECK(false);
+ ((void(*)())0)(); // crash
+ // force use of stored_token
+ stored_token = Token::ILLEGAL;
}
} }
Index: tools/lexer_generator/code_generator.py
diff --git a/tools/lexer_generator/code_generator.py
b/tools/lexer_generator/code_generator.py
index
b609c238b49a46f3fa3ac724c413ad0d9546d70a..03c830387d5603af8985a47f15844eb72777919e
100644
--- a/tools/lexer_generator/code_generator.py
+++ b/tools/lexer_generator/code_generator.py
@@ -131,6 +131,7 @@ class CodeGenerator:
'deferred_transitions' : [],
'long_char_transitions' : [],
'disjoint_keys' : disjoint_keys,
+ 'has_goto_after_entry' : False,
'inline' : None,
'depth' : None,
'action' : action,
@@ -288,6 +289,26 @@ class CodeGenerator:
assert len(dfa_states) == self.__dfa.node_count()
# store states
self.__dfa_states = dfa_states
+ return id_map
+
+ def __rewrite_gotos(self):
+ goto_map = {}
+ for state in self.__dfa_states:
+ if (state['match_action'] and
+ state['match_action'][0] == 'do_stored_token'):
+ assert not state['match_action'][1] in goto_map
+ goto_map[state['match_action'][1]] = state['node_number']
+ state['has_goto_after_entry'] = True
+ for state in self.__dfa_states:
+ if state['match_action'] and state['match_action'][0] == 'goto':
+ state['match_action'] = ('goto',
goto_map[state['match_action'][1]])
+
+ def process(self):
+
+ id_map = self.__canonicalize_traversal()
+ self.__rewrite_gotos()
+
+ dfa_states = self.__dfa_states
# set nodes to inline
if self.__inline:
inlined = reduce(self.__set_inline, dfa_states, 0)
@@ -303,12 +324,6 @@ class CodeGenerator:
for state in dfa_states:
self.__rewrite_deferred_transitions(state)
- def process(self):
-
- self.__canonicalize_traversal()
-
- dfa_states = self.__dfa_states
-
default_action = self.__default_action
assert(default_action and default_action.match_action())
default_action = default_action.match_action()
Index: tools/lexer_generator/dfa_optimizer.py
diff --git a/tools/lexer_generator/dfa_optimizer.py
b/tools/lexer_generator/dfa_optimizer.py
index
0b2c1e5136afe0e068d212f870449f11bb4afe80..2ac651da5ced87ca0b9236c4d0764c22a0658a5b
100644
--- a/tools/lexer_generator/dfa_optimizer.py
+++ b/tools/lexer_generator/dfa_optimizer.py
@@ -149,10 +149,10 @@ class DfaOptimizer(object):
assert not state_replacements
self.__dfa.visit_all_states(replace_state)
# now patch up all states with stores
- for state in store_states:
- old_action = states[state]['action']
- match_action = ('do_stored_token', None)
- states[state]['action'] = replacement_action(old_action,
match_action)
+ for state_id in store_states:
+ old_action = states[state_id]['action']
+ match_action = ('do_stored_token', state_id)
+ states[state_id]['action'] = replacement_action(old_action,
match_action)
start_name = name(self.__dfa.start_state())
if self.__log:
print 'gotos inserted %s' % counters['gotos']
Index: tools/lexer_generator/generator.py
diff --git a/tools/lexer_generator/generator.py
b/tools/lexer_generator/generator.py
index
1a1e5f36fb4b60fb5553007e8914c5666acc0712..c1efba7318c84bd2b1a28ae590b7b05062e25bf5
100644
--- a/tools/lexer_generator/generator.py
+++ b/tools/lexer_generator/generator.py
@@ -98,7 +98,7 @@ if __name__ == '__main__':
parser.add_argument('--input')
parser.add_argument('--code')
parser.add_argument('--encoding', default='latin1')
- parser.add_argument('--optimize-default', action='store_true')
+ parser.add_argument('--no-optimize-default', action='store_true')
parser.add_argument('--no-minimize-default', action='store_true')
parser.add_argument('--no-verify-default', action='store_true')
parser.add_argument('--no-inline', action='store_true')
@@ -115,7 +115,7 @@ if __name__ == '__main__':
with open(re_file, 'r') as f:
rule_processor = RuleProcessor.parse(f.read(), args.encoding)
- if args.optimize_default:
+ if not args.no_optimize_default:
rule_processor.default_automata().optimize_dfa(log = args.verbose)
if minimize_default:
--
--
v8-dev mailing list
[email protected]
http://groups.google.com/group/v8-dev
---
You received this message because you are subscribed to the Google Groups "v8-dev" group.
To unsubscribe from this group and stop receiving emails from it, send an email
to [email protected].
For more options, visit https://groups.google.com/groups/opt_out.
