[v8-dev] Re: Experimental lexer generator: random refactoring + comments. (issue 73023005)

Tue, 19 Nov 2013 10:26:13 -0800 

Reviewers: dcarney,

Message:
Committed patchset #2 manually as r17889.

Description:
Experimental lexer generator: random refactoring + comments.

BUG=
[email protected]

Committed: https://code.google.com/p/v8/source/detail?r=17889

Please review this at https://codereview.chromium.org/73023005/

SVN Base: https://v8.googlecode.com/svn/branches/experimental/parser

Affected files (+17, -12 lines):
  M tools/lexer_generator/code_generator.py
  M tools/lexer_generator/dfa.py
  M tools/lexer_generator/nfa.py
  M tools/lexer_generator/transition_keys.py


Index: tools/lexer_generator/code_generator.py
diff --git a/tools/lexer_generator/code_generator.py b/tools/lexer_generator/code_generator.py index 39129989cac49c26dbdba1f708799b1df0ed40ed..16dce4d79ddcf6bbbff5881436bce929d145a7c0 100644 
--- a/tools/lexer_generator/code_generator.py
+++ b/tools/lexer_generator/code_generator.py
@@ -148,6 +148,8 @@ class CodeGenerator:

   @staticmethod
   def __split_transitions(split_count, state):
+ '''Goes through the transitions for 'state' and decides which of them should 
+    use the if statement and which should use the switch statement.'''
     assert not state['switch_transitions']
     (class_keys, distinct_keys, ranges) = (0, 0, 0)
     for (t, r) in state['disjoint_keys']:
@@ -198,14 +200,14 @@ class CodeGenerator:
     if self.__inline:
       inlined = reduce(CodeGenerator.__set_inline, dfa_states, 0)
       if self.__log:
-        print "inlined %s" % inlined
+        print "%s states inlined" % inlined
     elif self.__log:
       print "no inlining"
     # split transitions
     if self.__switching:
       switched = reduce(CodeGenerator.__split_transitions, dfa_states, 0)
       if self.__log:
-        print "switched states %s" % inlined
+        print "%s states use switch (instead of if)" % switched
     elif self.__log:
       print "no switching"
     # store states
Index: tools/lexer_generator/dfa.py
diff --git a/tools/lexer_generator/dfa.py b/tools/lexer_generator/dfa.py
index d191e0518550f014d15076cf259b102dfd77786b..545fe84b6ee5d5e5531b89eabfdc84cc72c1dc23 100644 
--- a/tools/lexer_generator/dfa.py
+++ b/tools/lexer_generator/dfa.py
@@ -55,14 +55,17 @@ class DfaState(AutomatonState):

   # TODO abstract state matching
   def __matches(self, match_func, value):
- f = lambda acc, (k, vs): acc | set([vs]) if match_func(k, value) else acc 
+    # f collects states whose corresponding TransitionKey matches 'value'.
+    f = (lambda acc, (key, state):
+           acc | set([state]) if match_func(key, value) else acc)
     matches = reduce(f, self.__transitions.items(), set())
+ # Since this is a dfa, we should have at most one such state. Return it. 
     if not matches:
       return None
     assert len(matches) == 1
     return iter(matches).next()

-  def char_matches(self, value):
+  def next_state_with_char(self, value):
     return self.__matches(lambda k, v : k.matches_char(v), value)

   def key_matches(self, value):
Index: tools/lexer_generator/nfa.py
diff --git a/tools/lexer_generator/nfa.py b/tools/lexer_generator/nfa.py
index 24870e5e9036c90b8264b28526842bfa5a5b867f..8f9781dc476bb933462730253139377513f19c39 100644 
--- a/tools/lexer_generator/nfa.py
+++ b/tools/lexer_generator/nfa.py
@@ -98,10 +98,12 @@ class NfaState(AutomatonState):
       self.__add_transition(TransitionKey.epsilon(), end)

   def __matches(self, match_func, value):
-    f = lambda acc, (k, vs): acc | vs if match_func(k, value) else acc
+    # f collects states whose corresponding TransitionKey matches 'value'.
+    f = (lambda acc, (key, states):
+           acc | states if match_func(key, value) else acc)
     return reduce(f, self.__transitions.items(), set())

-  def char_matches(self, value):
+  def next_states_with_char(self, value):
     return self.__matches(lambda k, v : k.matches_char(v), value)

   def key_matches(self, value):
@@ -136,7 +138,7 @@ class Nfa(Automaton):
   def matches(self, string):
     valid_states = Nfa.__close(set([self.__start]))
     for c in string:
-      f = lambda acc, state: acc | state.char_matches(c)
+      f = lambda acc, state: acc | state.next_states_with_char(c)
       transitions = reduce(f, valid_states, set())
       if not transitions:
         return False
Index: tools/lexer_generator/transition_keys.py
diff --git a/tools/lexer_generator/transition_keys.py b/tools/lexer_generator/transition_keys.py index d0eb5b1f16d069ebe5862b749aa907f41ab02ba0..ba0d9db40cb615605929d0619f752d0326d7a60a 100644 
--- a/tools/lexer_generator/transition_keys.py
+++ b/tools/lexer_generator/transition_keys.py
@@ -103,11 +103,9 @@ class TransitionKey:
   @staticmethod
   def any():
     '''Returns a TransitionKey which matches any character.'''
-    def bounds_getter(name):
-      bounds = TransitionKey.__class_bounds.values()
-      bounds.sort()
-      return bounds
-    return TransitionKey.__cached_key('any', bounds_getter)
+    return TransitionKey.__cached_key(
+        'any',
+        lambda name : sorted(TransitionKey.__class_bounds.values()))

   @staticmethod
   def single_char(char):


--
--
v8-dev mailing list
[email protected]
http://groups.google.com/group/v8-dev
--- You received this message because you are subscribed to the Google Groups "v8-dev" group. 
To unsubscribe from this group and stop receiving emails from it, send an email 
to [email protected].
For more options, visit https://groups.google.com/groups/opt_out.

Reply via email to