Revision: 19295
Author: [email protected]
Date: Tue Feb 11 18:23:07 2014 UTC
Log: Experimental parser: store literals as ints
[email protected]
BUG=
Review URL: https://codereview.chromium.org/157813004
http://code.google.com/p/v8/source/detail?r=19295
Modified:
/branches/experimental/parser/tools/lexer_generator/automata_test.py
/branches/experimental/parser/tools/lexer_generator/nfa_builder.py
/branches/experimental/parser/tools/lexer_generator/regex_parser.py
/branches/experimental/parser/tools/lexer_generator/transition_key_test.py
/branches/experimental/parser/tools/lexer_generator/transition_keys.py
=======================================
--- /branches/experimental/parser/tools/lexer_generator/automata_test.py
Tue Feb 11 12:22:19 2014 UTC
+++ /branches/experimental/parser/tools/lexer_generator/automata_test.py
Tue Feb 11 18:23:07 2014 UTC
@@ -91,9 +91,9 @@
'terminal' : False,
'action' : Action.empty_action() }
mapping = { k : empty_node() for k in ['S_0', 'S_1', 'S_2', 'S_3'] }
- key_a = TransitionKey.single_char(encoding, 'a')
- key_b = TransitionKey.single_char(encoding, 'b')
- key_c = TransitionKey.single_char(encoding, 'c')
+ key_a = TransitionKey.single_char(encoding, ord('a'))
+ key_b = TransitionKey.single_char(encoding, ord('b'))
+ key_c = TransitionKey.single_char(encoding, ord('c'))
mapping['S_0']['transitions'][key_a] = 'S_1'
mapping['S_0']['transitions'][key_b] = 'S_2'
=======================================
--- /branches/experimental/parser/tools/lexer_generator/nfa_builder.py Fri
Feb 7 08:31:41 2014 UTC
+++ /branches/experimental/parser/tools/lexer_generator/nfa_builder.py Tue
Feb 11 18:23:07 2014 UTC
@@ -128,7 +128,7 @@
state.add_unclosed_transition(key)
return (state, [state])
- def __literal(self, chars):
+ def __literal(self, *chars):
terms = map(lambda c : Term('SINGLE_CHAR', c), chars)
return self.__process(self.cat_terms(terms))
@@ -320,24 +320,21 @@
@staticmethod
def __flatten_literals(terms):
- literal = None
+ acc = ()
for term in terms:
assert isinstance(term, Term)
if not term:
continue
if term.name() == 'LITERAL':
- if literal:
- literal += term.args()[0]
- else:
- literal = term.args()[0]
+ acc += term.args()
else:
- if literal:
- yield Term('LITERAL', literal)
- literal = None
+ if acc:
+ yield Term('LITERAL', *acc)
+ acc = ()
if term:
yield term
- if literal:
- yield Term('LITERAL', literal)
+ if acc:
+ yield Term('LITERAL', *acc)
@staticmethod
def or_terms(terms):
=======================================
--- /branches/experimental/parser/tools/lexer_generator/regex_parser.py Tue
Feb 11 12:22:19 2014 UTC
+++ /branches/experimental/parser/tools/lexer_generator/regex_parser.py Tue
Feb 11 18:23:07 2014 UTC
@@ -238,7 +238,7 @@
def p_literal(self, p):
'''literal : LITERAL'''
- p[0] = Term('LITERAL', p[1])
+ p[0] = Term('LITERAL', ord(p[1]))
def p_any(self, p):
'''any : ANY'''
@@ -262,10 +262,10 @@
| CHARACTER_CLASS maybe_class_content
'''
if len(p) == 5:
- left = Term("RANGE", p[1], p[3])
+ left = Term("RANGE", ord(p[1]), ord(p[3]))
else:
if len(p[1]) == 1:
- left = Term('LITERAL', p[1])
+ left = Term('LITERAL', ord(p[1]))
else:
left = Term('CHARACTER_CLASS', p[1][1:-1])
p[0] = self.__cat(left, p[len(p)-1])
=======================================
---
/branches/experimental/parser/tools/lexer_generator/transition_key_test.py
Tue Feb 11 12:22:19 2014 UTC
+++
/branches/experimental/parser/tools/lexer_generator/transition_key_test.py
Tue Feb 11 18:23:07 2014 UTC
@@ -36,8 +36,8 @@
__equal_pairs = [
(TransitionKey.epsilon(), TransitionKey.epsilon()),
(TransitionKey.any(__encoding), TransitionKey.any(__encoding)),
- (TransitionKey.single_char(__encoding, 'a'),
- TransitionKey.single_char(__encoding, 'a')),
+ (TransitionKey.single_char(__encoding, ord('a')),
+ TransitionKey.single_char(__encoding, ord('a'))),
]
def test_eq(self):
=======================================
--- /branches/experimental/parser/tools/lexer_generator/transition_keys.py
Tue Feb 11 12:22:19 2014 UTC
+++ /branches/experimental/parser/tools/lexer_generator/transition_keys.py
Tue Feb 11 18:23:07 2014 UTC
@@ -25,6 +25,7 @@
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+from types import IntType
from itertools import chain
from action import Term
from string import printable
@@ -144,8 +145,7 @@
@staticmethod
def omega():
'''Always matches.'''
- return TransitionKey.__cached_key(None, 'omega',
- lambda : Term("OMEGA_KEY"))
+ return TransitionKey.__cached_key(None, 'omega', lambda :
Term("OMEGA_KEY"))
@staticmethod
def any(encoding):
@@ -154,13 +154,14 @@
lambda : encoding.all_components_iter())
@staticmethod
- def single_char(encoding, char): # TODO(dcarney): char should be int
+ def single_char(encoding, char):
'''Returns a TransitionKey for a single-character transition.'''
- return TransitionKey(encoding, Term("NUMERIC_RANGE_KEY", ord(char),
ord(char)))
+ return TransitionKey.range(encoding, char, char)
@staticmethod
def range(encoding, a, b):
'''Returns a TransitionKey for a single-character transition.'''
+ assert type(a) == IntType and type(b) == IntType
return TransitionKey(encoding, Term("NUMERIC_RANGE_KEY", a, b))
@staticmethod
@@ -176,10 +177,9 @@
key = term.name()
args = term.args()
if key == 'RANGE':
- components.append(Term('NUMERIC_RANGE_KEY', ord(args[0]),
ord(args[1])))
+ components.append(Term('NUMERIC_RANGE_KEY', args[0], args[1]))
elif key == 'LITERAL':
- for char in args[0]: # TODO(dcarney): don't use strings for literals
- components.append(Term('NUMERIC_RANGE_KEY', ord(char), ord(char)))
+ components += map(lambda x : Term('NUMERIC_RANGE_KEY', x, x), args)
elif key == 'CAT':
for x in args:
TransitionKey.__process_term(encoding, x, components, key_map)
--
--
v8-dev mailing list
[email protected]
http://groups.google.com/group/v8-dev
---
You received this message because you are subscribed to the Google Groups "v8-dev" group.
To unsubscribe from this group and stop receiving emails from it, send an email
to [email protected].
For more options, visit https://groups.google.com/groups/opt_out.
