Revision: 17728
Author: [email protected]
Date: Thu Nov 14 10:46:34 2013 UTC
Log: Experimental parser: add push_token syntax
[email protected]
BUG=
Review URL: https://codereview.chromium.org/63773003
http://code.google.com/p/v8/source/detail?r=17728
Modified:
/branches/experimental/parser/src/lexer/lexer_py.re
/branches/experimental/parser/tools/lexer_generator/code_generator.py
/branches/experimental/parser/tools/lexer_generator/rule_lexer.py
/branches/experimental/parser/tools/lexer_generator/rule_parser.py
=======================================
--- /branches/experimental/parser/src/lexer/lexer_py.re Thu Nov 14 09:47:12
2013 UTC
+++ /branches/experimental/parser/src/lexer/lexer_py.re Thu Nov 14 10:46:34
2013 UTC
@@ -36,21 +36,21 @@
number = ("0x" hex_digit+) | (("." digit+ maybe_exponent) | (digit+ ("."
digit*)? maybe_exponent));
<default>
-"|=" { PUSH_TOKEN(Token::ASSIGN_BIT_OR); }
-"^=" { PUSH_TOKEN(Token::ASSIGN_BIT_XOR); }
-"&=" { PUSH_TOKEN(Token::ASSIGN_BIT_AND); }
-"+=" { PUSH_TOKEN(Token::ASSIGN_ADD); }
-"-=" { PUSH_TOKEN(Token::ASSIGN_SUB); }
-"*=" { PUSH_TOKEN(Token::ASSIGN_MUL); }
-"/=" { PUSH_TOKEN(Token::ASSIGN_DIV); }
-"%=" { PUSH_TOKEN(Token::ASSIGN_MOD); }
+"|=" push_token(ASSIGN_BIT_OR)
+"^=" push_token(ASSIGN_BIT_XOR)
+"&=" push_token(ASSIGN_BIT_AND)
+"+=" push_token(ASSIGN_ADD)
+"-=" push_token(ASSIGN_SUB)
+"*=" push_token(ASSIGN_MUL)
+"/=" push_token(ASSIGN_DIV)
+"%=" push_token(ASSIGN_MOD)
-"===" { PUSH_TOKEN(Token::EQ_STRICT); }
-"==" { PUSH_TOKEN(Token::EQ); }
-"=" { PUSH_TOKEN(Token::ASSIGN); }
-"!==" { PUSH_TOKEN(Token::NE_STRICT); }
-"!=" { PUSH_TOKEN(Token::NE); }
-"!" { PUSH_TOKEN(Token::NOT); }
+"===" push_token(EQ_STRICT)
+"==" push_token(EQ)
+"=" push_token(ASSIGN)
+"!==" push_token(NE_STRICT)
+"!=" push_token(NE)
+"!" push_token(NOT)
"//" <<SingleLineComment>>
"/*" <<MultiLineComment>>
@@ -58,46 +58,46 @@
#whitespace* "-->" { if (just_seen_line_terminator_) {
YYSETCONDITION(kConditionSingleLineComment); goto yyc_SingleLineComment; }
else { --cursor_; send(Token::DEC); start_ = cursor_; goto yyc_Normal; } }
-">>>=" { PUSH_TOKEN(Token::ASSIGN_SHR); }
-">>>" { PUSH_TOKEN(Token::SHR); }
-"<<=" { PUSH_TOKEN(Token::ASSIGN_SHL); }
-">>=" { PUSH_TOKEN(Token::ASSIGN_SAR); }
-"<=" { PUSH_TOKEN(Token::LTE); }
-">=" { PUSH_TOKEN(Token::GTE); }
-"<<" { PUSH_TOKEN(Token::SHL); }
-">>" { PUSH_TOKEN(Token::SAR); }
-"<" { PUSH_TOKEN(Token::LT); }
-">" { PUSH_TOKEN(Token::GT); }
+">>>=" push_token(ASSIGN_SHR)
+">>>" push_token(SHR)
+"<<=" push_token(ASSIGN_SHL)
+">>=" push_token(ASSIGN_SAR)
+"<=" push_token(LTE)
+">=" push_token(GTE)
+"<<" push_token(SHL)
+">>" push_token(SAR)
+"<" push_token(LT)
+">" push_token(GT)
-number { PUSH_TOKEN(Token::NUMBER); }
-# number identifier_char { PUSH_TOKEN(Token::ILLEGAL); }
+number push_token(NUMBER)
+# number identifier_char push_token(ILLEGAL)
-"(" { PUSH_TOKEN(Token::LPAREN); }
-")" { PUSH_TOKEN(Token::RPAREN); }
-"[" { PUSH_TOKEN(Token::LBRACK); }
-"]" { PUSH_TOKEN(Token::RBRACK); }
-"{" { PUSH_TOKEN(Token::LBRACE); }
-"}" { PUSH_TOKEN(Token::RBRACE); }
-":" { PUSH_TOKEN(Token::COLON); }
-";" { PUSH_TOKEN(Token::SEMICOLON); }
-"." { PUSH_TOKEN(Token::PERIOD); }
-"?" { PUSH_TOKEN(Token::CONDITIONAL); }
-"++" { PUSH_TOKEN(Token::INC); }
-"--" { PUSH_TOKEN(Token::DEC); }
+"(" push_token(LPAREN)
+")" push_token(RPAREN)
+"[" push_token(LBRACK)
+"]" push_token(RBRACK)
+"{" push_token(LBRACE)
+"}" push_token(RBRACE)
+":" push_token(COLON)
+";" push_token(SEMICOLON)
+"." push_token(PERIOD)
+"?" push_token(CONDITIONAL)
+"++" push_token(INC)
+"--" push_token(DEC)
-"||" { PUSH_TOKEN(Token::OR); }
-"&&" { PUSH_TOKEN(Token::AND); }
+"||" push_token(OR)
+"&&" push_token(AND)
-"|" { PUSH_TOKEN(Token::BIT_OR); }
-"^" { PUSH_TOKEN(Token::BIT_XOR); }
-"&" { PUSH_TOKEN(Token::BIT_AND); }
-"+" { PUSH_TOKEN(Token::ADD); }
-"-" { PUSH_TOKEN(Token::SUB); }
-"*" { PUSH_TOKEN(Token::MUL); }
-"/" { PUSH_TOKEN(Token::DIV); }
-"%" { PUSH_TOKEN(Token::MOD); }
-"~" { PUSH_TOKEN(Token::BIT_NOT); }
-"," { PUSH_TOKEN(Token::COMMA); }
+"|" push_token(BIT_OR)
+"^" push_token(BIT_XOR)
+"&" push_token(BIT_AND)
+"+" push_token(ADD)
+"-" push_token(SUB)
+"*" push_token(MUL)
+"/" push_token(DIV)
+"%" push_token(MOD)
+"~" push_token(BIT_NOT)
+"," push_token(COMMA)
line_terminator+ { PUSH_LINE_TERMINATOR(); }
whitespace { SKIP(); } # TODO implement skip
@@ -106,53 +106,53 @@
"'" <<SingleQuoteString>>
# all keywords
-"break" { PUSH_TOKEN(Token::BREAK); }
-"case" { PUSH_TOKEN(Token::CASE); }
-"catch" { PUSH_TOKEN(Token::CATCH); }
-"class" { PUSH_TOKEN(Token::FUTURE_RESERVED_WORD); }
-"const" { PUSH_TOKEN(Token::CONST); }
-"continue" { PUSH_TOKEN(Token::CONTINUE); }
-"debugger" { PUSH_TOKEN(Token::DEBUGGER); }
-"default" { PUSH_TOKEN(Token::DEFAULT); }
-"delete" { PUSH_TOKEN(Token::DELETE); }
-"do" { PUSH_TOKEN(Token::DO); }
-"else" { PUSH_TOKEN(Token::ELSE); }
-"enum" { PUSH_TOKEN(Token::FUTURE_RESERVED_WORD); }
-"export" { PUSH_TOKEN(Token::FUTURE_RESERVED_WORD); }
-"extends" { PUSH_TOKEN(Token::FUTURE_RESERVED_WORD); }
-"false" { PUSH_TOKEN(Token::FALSE_LITERAL); }
-"finally" { PUSH_TOKEN(Token::FINALLY); }
-"for" { PUSH_TOKEN(Token::FOR); }
-"function" { PUSH_TOKEN(Token::FUNCTION); }
-"if" { PUSH_TOKEN(Token::IF); }
-"implements" { PUSH_TOKEN(Token::FUTURE_STRICT_RESERVED_WORD); }
-"import" { PUSH_TOKEN(Token::FUTURE_RESERVED_WORD); }
-"in" { PUSH_TOKEN(Token::IN); }
-"instanceof" { PUSH_TOKEN(Token::INSTANCEOF); }
-"interface" { PUSH_TOKEN(Token::FUTURE_STRICT_RESERVED_WORD); }
-"let" { PUSH_TOKEN(Token::FUTURE_STRICT_RESERVED_WORD); }
-"new" { PUSH_TOKEN(Token::NEW); }
-"null" { PUSH_TOKEN(Token::NULL_LITERAL); }
-"package" { PUSH_TOKEN(Token::FUTURE_STRICT_RESERVED_WORD); }
-"private" { PUSH_TOKEN(Token::FUTURE_STRICT_RESERVED_WORD); }
-"protected" { PUSH_TOKEN(Token::FUTURE_STRICT_RESERVED_WORD); }
-"public" { PUSH_TOKEN(Token::FUTURE_STRICT_RESERVED_WORD); }
-"return" { PUSH_TOKEN(Token::RETURN); }
-"static" { PUSH_TOKEN(Token::FUTURE_STRICT_RESERVED_WORD); }
-"super" { PUSH_TOKEN(Token::FUTURE_RESERVED_WORD); }
-"switch" { PUSH_TOKEN(Token::SWITCH); }
-"this" { PUSH_TOKEN(Token::THIS); }
-"throw" { PUSH_TOKEN(Token::THROW); }
-"true" { PUSH_TOKEN(Token::TRUE_LITERAL); }
-"try" { PUSH_TOKEN(Token::TRY); }
-"typeof" { PUSH_TOKEN(Token::TYPEOF); }
-"var" { PUSH_TOKEN(Token::VAR); }
-"void" { PUSH_TOKEN(Token::VOID); }
-"while" { PUSH_TOKEN(Token::WHILE); }
-"with" { PUSH_TOKEN(Token::WITH); }
-"yield" { PUSH_TOKEN(Token::YIELD); }
+"break" push_token(BREAK)
+"case" push_token(CASE)
+"catch" push_token(CATCH)
+"class" push_token(FUTURE_RESERVED_WORD)
+"const" push_token(CONST)
+"continue" push_token(CONTINUE)
+"debugger" push_token(DEBUGGER)
+"default" push_token(DEFAULT)
+"delete" push_token(DELETE)
+"do" push_token(DO)
+"else" push_token(ELSE)
+"enum" push_token(FUTURE_RESERVED_WORD)
+"export" push_token(FUTURE_RESERVED_WORD)
+"extends" push_token(FUTURE_RESERVED_WORD)
+"false" push_token(FALSE_LITERAL)
+"finally" push_token(FINALLY)
+"for" push_token(FOR)
+"function" push_token(FUNCTION)
+"if" push_token(IF)
+"implements" push_token(FUTURE_STRICT_RESERVED_WORD)
+"import" push_token(FUTURE_RESERVED_WORD)
+"in" push_token(IN)
+"instanceof" push_token(INSTANCEOF)
+"interface" push_token(FUTURE_STRICT_RESERVED_WORD)
+"let" push_token(FUTURE_STRICT_RESERVED_WORD)
+"new" push_token(NEW)
+"null" push_token(NULL_LITERAL)
+"package" push_token(FUTURE_STRICT_RESERVED_WORD)
+"private" push_token(FUTURE_STRICT_RESERVED_WORD)
+"protected" push_token(FUTURE_STRICT_RESERVED_WORD)
+"public" push_token(FUTURE_STRICT_RESERVED_WORD)
+"return" push_token(RETURN)
+"static" push_token(FUTURE_STRICT_RESERVED_WORD)
+"super" push_token(FUTURE_RESERVED_WORD)
+"switch" push_token(SWITCH)
+"this" push_token(THIS)
+"throw" push_token(THROW)
+"true" push_token(TRUE_LITERAL)
+"try" push_token(TRY)
+"typeof" push_token(TYPEOF)
+"var" push_token(VAR)
+"void" push_token(VOID)
+"while" push_token(WHILE)
+"with" push_token(WITH)
+"yield" push_token(YIELD)
-identifier_start { PUSH_TOKEN(Token::IDENTIFIER); } <<Identifier>>
+identifier_start push_token(IDENTIFIER) <<Identifier>>
/\\u[0-9a-fA-F]{4}/ {
if (V8_UNLIKELY(!ValidIdentifierStart())) {
PUSH_TOKEN(Token::ILLEGAL);
@@ -160,14 +160,14 @@
} <<Identifier>>
eof { PUSH_TOKEN(Token::EOS); return 0; }
-default_action { PUSH_TOKEN(Token::ILLEGAL); }
+default_action push_token(ILLEGAL)
<DoubleQuoteString>
/\\\n\r?/ <<continue>>
/\\\r\n?/ <<continue>>
/\\./ <<continue>>
-/\n|\r/ { PUSH_TOKEN(Token::ILLEGAL); }
-"\"" { PUSH_TOKEN(Token::STRING); }
+/\n|\r/ push_token(ILLEGAL)
+"\"" push_token(STRING)
eof <<terminate_illegal>>
catch_all <<continue>>
@@ -175,8 +175,8 @@
/\\\n\r?/ <<continue>>
/\\\r\n?/ <<continue>>
/\\./ <<continue>>
-/\n|\r/ { PUSH_TOKEN(Token::ILLEGAL); }
-"'" { PUSH_TOKEN(Token::STRING); }
+/\n|\r/ push_token(ILLEGAL)
+"'" push_token(STRING)
eof <<terminate_illegal>>
catch_all <<continue>>
@@ -187,7 +187,7 @@
PUSH_TOKEN(Token::ILLEGAL);
}
} <<continue>>
-default_action { PUSH_TOKEN(Token::IDENTIFIER); }
+default_action push_token(IDENTIFIER)
<SingleLineComment>
line_terminator { PUSH_LINE_TERMINATOR(); }
=======================================
--- /branches/experimental/parser/tools/lexer_generator/code_generator.py
Thu Nov 14 07:25:37 2013 UTC
+++ /branches/experimental/parser/tools/lexer_generator/code_generator.py
Thu Nov 14 10:46:34 2013 UTC
@@ -88,8 +88,14 @@
''' % s.node_number()
if action:
- code += '%s\nBACK();\ngoto code_%s;\n' % (action.data(),
- start_node_number)
+ if action.type() == 'code':
+ code += '%s\nBACK();\ngoto code_%s;\n' % (action.data(),
+ start_node_number)
+ elif action.type() == 'push_token':
+ #TODO
+ pass
+ else:
+ raise Exception("unknown type %s" % action.type())
else:
code += 'goto default_action;'
return code
=======================================
--- /branches/experimental/parser/tools/lexer_generator/rule_lexer.py Tue
Nov 12 07:12:31 2013 UTC
+++ /branches/experimental/parser/tools/lexer_generator/rule_lexer.py Thu
Nov 14 10:46:34 2013 UTC
@@ -33,6 +33,7 @@
'DEFAULT',
'DEFAULT_ACTION',
'CATCH_ALL',
+ 'PUSH_TOKEN',
'IDENTIFIER',
'STRING',
@@ -70,7 +71,7 @@
pass
__special_identifiers = set(map(lambda s: s.lower(),
- ['DEFAULT', 'DEFAULT_ACTION', 'CATCH_ALL',]))
+ ['DEFAULT', 'DEFAULT_ACTION', 'CATCH_ALL', 'PUSH_TOKEN']))
def t_IDENTIFIER(self, t):
r'[a-zA-Z][a-zA-Z0-9_]*'
=======================================
--- /branches/experimental/parser/tools/lexer_generator/rule_parser.py Thu
Nov 14 10:16:32 2013 UTC
+++ /branches/experimental/parser/tools/lexer_generator/rule_parser.py Thu
Nov 14 10:46:34 2013 UTC
@@ -98,10 +98,10 @@
| empty'''
def p_transition_rule(self, p):
- '''transition_rule : composite_regex code action
+ '''transition_rule : composite_regex code_or_token action
| composite_regex empty action
- | composite_regex code empty
- | DEFAULT_ACTION code empty
+ | composite_regex code_or_token empty
+ | DEFAULT_ACTION code_or_token empty
| CATCH_ALL empty action'''
transition = p[3]
if transition and not transition in self.__keyword_transitions:
@@ -120,6 +120,15 @@
rule = (p[1], RuleParser.__rule_precedence_counter, code, transition)
rules['regex'].append(rule)
+ def p_code_or_token(self, p):
+ '''code_or_token : code
+ | push_token'''
+ p[0] = p[1]
+
+ def p_push_token(self, p):
+ 'push_token : PUSH_TOKEN LEFT_PARENTHESIS IDENTIFIER RIGHT_PARENTHESIS'
+ p[0] = (p[1], p[3])
+
def p_action(self, p):
'action : ACTION_OPEN IDENTIFIER ACTION_CLOSE'
p[0] = p[2]
@@ -178,7 +187,7 @@
def p_code(self, p):
'code : LEFT_BRACKET code_fragments RIGHT_BRACKET'
- p[0] = p[2].strip()
+ p[0] = ('code', p[2].strip())
def p_code_fragments(self, p):
'''code_fragments : CODE_FRAGMENT code_fragments
@@ -269,7 +278,8 @@
for (graph, precedence, code, transition) in v['regex']:
default_code = v['default_action']
if code or default_code:
- action = Action('code', code if code else default_code,
precedence)
+ (code_type, code_value) = code if code else default_code
+ action = Action(code_type, code_value, precedence)
graph = NfaBuilder.add_action(graph, action)
if not transition or transition == 'break':
pass
--
--
v8-dev mailing list
[email protected]
http://groups.google.com/group/v8-dev
---
You received this message because you are subscribed to the Google Groups "v8-dev" group.
To unsubscribe from this group and stop receiving emails from it, send an email
to [email protected].
For more options, visit https://groups.google.com/groups/opt_out.
