Reviewers: ,
Message:
Committed patchset #1 manually as r17546 (presubmit successful).
Description:
Experimental lexer generator: add tests for actions.
[email protected]
BUG=
Committed: https://code.google.com/p/v8/source/detail?r=17546
Please review this at https://codereview.chromium.org/64073002/
SVN Base: https://v8.googlecode.com/svn/branches/experimental/parser
Affected files (+92, -1 lines):
A tools/lexer_generator/action_test.py
M tools/lexer_generator/test_suite.py
Index: tools/lexer_generator/action_test.py
diff --git a/tools/lexer_generator/action_test.py
b/tools/lexer_generator/action_test.py
new file mode 100644
index
0000000000000000000000000000000000000000..bb80100f83e8f49a005a00d91a8cb91c362f3b3f
--- /dev/null
+++ b/tools/lexer_generator/action_test.py
@@ -0,0 +1,89 @@
+# Copyright 2013 the V8 project authors. All rights reserved.
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions are
+# met:
+#
+# * Redistributions of source code must retain the above copyright
+# notice, this list of conditions and the following disclaimer.
+# * Redistributions in binary form must reproduce the above
+# copyright notice, this list of conditions and the following
+# disclaimer in the documentation and/or other materials provided
+# with the distribution.
+# * Neither the name of Google Inc. nor the names of its
+# contributors may be used to endorse or promote products derived
+# from this software without specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+import unittest
+from rule_parser import RuleParser, RuleParserState
+from nfa import NfaBuilder
+from dfa import Dfa
+
+def dfa_from_nfa(nfa):
+ (start_name, dfa_nodes) = nfa.compute_dfa()
+ return Dfa(start_name, dfa_nodes)
+
+def process_rules(parser_state):
+ rule_map = {}
+ builder = NfaBuilder()
+ for k, v in parser_state.rules.items():
+ graphs = []
+ for (graph, precedence, code, condition) in v['regex']:
+ graphs.append(NfaBuilder.add_action(graph, (precedence, code,
condition)))
+ nfa = builder.nfa(NfaBuilder.or_graphs(graphs))
+ dfa = dfa_from_nfa(nfa)
+ rule_map[k] = (nfa, dfa)
+ return rule_map
+
+class ActionTestCase(unittest.TestCase):
+
+ def __verify_last_action(self, dfa, string, expected_code,
+ expected_condition):
+ actions = list(dfa.collect_actions(string))
+ self.assertEqual(actions[-1], ('TERMINATE',))
+ self.assertEqual(actions[-2][1], expected_code)
+ self.assertEqual(actions[-2][2], expected_condition)
+
+ def test_action_precedence(self):
+ parser_state = RuleParserState()
+ rules = '''<default>
+ "key" { KEYWORD } <<break>>
+ /[a-z]+/ { ID } <<break>>'''
+ RuleParser.parse(rules, parser_state)
+ automata_for_conditions = process_rules(parser_state)
+ self.assertEqual(len(automata_for_conditions), 1)
+ self.assertTrue('default' in automata_for_conditions)
+ (nfa, dfa) = automata_for_conditions['default']
+
+ self.__verify_last_action(dfa, 'foo', 'ID', 'break')
+ self.__verify_last_action(dfa, 'key', 'KEYWORD', 'break')
+ self.__verify_last_action(dfa, 'k', 'ID', 'break')
+ self.__verify_last_action(dfa, 'ke', 'ID', 'break')
+ self.__verify_last_action(dfa, 'keys', 'ID', 'break')
+
+ def test_wrong_action_precedence(self):
+ parser_state = RuleParserState()
+ rules = '''<default>
+ /[a-z]+/ { ID } <<break>>
+ "key" { KEYWORD } <<break>>'''
+ RuleParser.parse(rules, parser_state)
+ automata_for_conditions = process_rules(parser_state)
+ self.assertEqual(len(automata_for_conditions), 1)
+ self.assertTrue('default' in automata_for_conditions)
+ (nfa, dfa) = automata_for_conditions['default']
+
+ # The keyword is not recognized because of the rule preference order
(ID
+ # is preferred over KEYWORD).
+ self.__verify_last_action(dfa, 'foo', 'ID', 'break')
+ self.__verify_last_action(dfa, 'key', 'ID', 'break')
Index: tools/lexer_generator/test_suite.py
diff --git a/tools/lexer_generator/test_suite.py
b/tools/lexer_generator/test_suite.py
index
18de9951634aa82ad23061bf8026c89a4af7271b..4aec610e8baea9f9764618a85cce16d7f3b6aa9c
100644
--- a/tools/lexer_generator/test_suite.py
+++ b/tools/lexer_generator/test_suite.py
@@ -27,9 +27,10 @@
from unittest import TestLoader, TextTestRunner, TestSuite
-from transition_key_test import *
+from action_test import *
from automata_test import *
from rule_parser_test import *
+from transition_key_test import *
if __name__ == "__main__":
loader = TestLoader()
@@ -37,6 +38,7 @@ if __name__ == "__main__":
loader.loadTestsFromTestCase(TransitionKeyTestCase),
loader.loadTestsFromTestCase(AutomataTestCase),
loader.loadTestsFromTestCase(RuleParserTestCase),
+ loader.loadTestsFromTestCase(ActionTestCase),
))
runner = TextTestRunner(verbosity = 2)
runner.run(suite)
--
--
v8-dev mailing list
[email protected]
http://groups.google.com/group/v8-dev
---
You received this message because you are subscribed to the Google Groups "v8-dev" group.
To unsubscribe from this group and stop receiving emails from it, send an email
to [email protected].
For more options, visit https://groups.google.com/groups/opt_out.
