Reviewers: dcarney,
Message:
Committed patchset #1 manually as r17618.
Description:
Experimental lexer generator: Code generator skeleton.
[email protected]
BUG=
Committed: https://code.google.com/p/v8/source/detail?r=17618
Please review this at https://codereview.chromium.org/68803002/
SVN Base: https://v8.googlecode.com/svn/branches/experimental/parser
Affected files (+57, -5 lines):
M tools/lexer_generator/dfa.py
M tools/lexer_generator/generator.py
M tools/lexer_generator/transition_keys.py
Index: tools/lexer_generator/dfa.py
diff --git a/tools/lexer_generator/dfa.py b/tools/lexer_generator/dfa.py
index
95590f2cadbf3e90e7f9bcaffc9493cf3f173c67..73e8eb4c26c35c1a564cd37b410e4828d8b59e27
100644
--- a/tools/lexer_generator/dfa.py
+++ b/tools/lexer_generator/dfa.py
@@ -53,19 +53,36 @@ class DfaState(AutomatonState):
def transitions(self):
return self.__transitions
+ def to_code(self):
+ # FIXME: add different check types (if, switch, lookup table)
+ # FIXME: add action + break / continue
+ # FIXME: add default action
+ code = '''
+code_%s:
+fprintf(stderr, "state %s, char at hand is %%c\\n", c);
+''' % (self.node_number(), self.node_number())
+
+ for key, state in self.__transitions.items():
+ code += key.to_code()
+ code += ''' {
+ c = *(++cursor);
+ goto code_%s;
+}''' % state.node_number()
+ return code
+
class Dfa(Automaton):
def __init__(self, start_name, mapping):
super(Dfa, self).__init__()
self.__terminal_set = set()
- name_map = {}
+ self.__name_map = {}
for i, (name, node_data) in enumerate(mapping.items()):
node = DfaState(name, i, node_data['actions'])
- name_map[name] = node
+ self.__name_map[name] = node
if node_data['terminal']:
self.__terminal_set.add(node)
for name, node_data in mapping.items():
- node = name_map[name]
+ node = self.__name_map[name]
inversion = {}
for key, state in node_data['transitions'].items():
if not state in inversion:
@@ -73,8 +90,8 @@ class Dfa(Automaton):
inversion[state].append(key)
for state, keys in inversion.items():
merged_key = TransitionKey.merged_key(keys)
- node.add_transition(merged_key, name_map[state])
- self.__start = name_map[start_name]
+ node.add_transition(merged_key, self.__name_map[state])
+ self.__start = self.__name_map[start_name]
assert self.__terminal_set
@staticmethod
@@ -127,3 +144,12 @@ class Dfa(Automaton):
iterator = lambda visitor, state: self.__visit_all_edges(visitor,
state)
state_iterator = lambda x : [x]
return self.generate_dot(self.__start, self.__terminal_set, iterator,
state_iterator)
+
+ def to_code(self):
+ code = '''
+char c = *cursor;
+goto code_%s;
+''' % (self.__start.node_number())
+ for n in self.__name_map.values():
+ code += n.to_code()
+ return code
Index: tools/lexer_generator/generator.py
diff --git a/tools/lexer_generator/generator.py
b/tools/lexer_generator/generator.py
index
7f929cd14bfd7e1b508ddf977c43ea81bb40793f..396a7bafdd42c30fcbe17564f3f205d4182f1678
100644
--- a/tools/lexer_generator/generator.py
+++ b/tools/lexer_generator/generator.py
@@ -134,12 +134,17 @@ class Generator(object):
(nfa, dfa) = self.__automata['default']
return dfa.lex(string)
+ def generate_code(self):
+ (nfa, dfa) = self.__automata['default']
+ return dfa.to_code()
+
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--html')
parser.add_argument('--re', default='src/lexer/lexer_py.re')
parser.add_argument('--input')
+ parser.add_argument('--code')
args = parser.parse_args()
re_file = args.re
@@ -155,6 +160,13 @@ if __name__ == '__main__':
f.write(html)
print "wrote html to %s" % html_file
+ code_file = args.code
+ if code_file:
+ code = generator.generate_code()
+ with open(code_file, 'w') as f:
+ f.write(code)
+ print "wrote code to %s" % code_file
+
input_file = args.input
if input_file:
with open(input_file, 'r') as f:
Index: tools/lexer_generator/transition_keys.py
diff --git a/tools/lexer_generator/transition_keys.py
b/tools/lexer_generator/transition_keys.py
index
b18cd1085122a856e9c1146f311cd8560b011219..1d9fbfbec5b3dda7dac57b8c0608f5a60a95c5a3
100644
--- a/tools/lexer_generator/transition_keys.py
+++ b/tools/lexer_generator/transition_keys.py
@@ -175,6 +175,20 @@ class TransitionKey:
def __eq__(self, other):
return isinstance(other, self.__class__) and self.__ranges ==
other.__ranges
+ def to_code(self):
+ code = 'if ('
+ first = True
+ for r in self.__ranges:
+ if not first:
+ code += ' || '
+ if r[0] == r[1]:
+ code += 'c == %s' % r[0]
+ else:
+ code += '(c >= %s && c <= %s)' % (r[0], r[1])
+ first = False
+ code += ')'
+ return code
+
__printable_cache = {
ord('\t') : '\\t',
ord('\n') : '\\n',
--
--
v8-dev mailing list
[email protected]
http://groups.google.com/group/v8-dev
---
You received this message because you are subscribed to the Google Groups "v8-dev" group.
To unsubscribe from this group and stop receiving emails from it, send an email
to [email protected].
For more options, visit https://groups.google.com/groups/opt_out.
