felipe Sun, 11 Jul 2010 14:17:26 +0000 Revision: http://svn.php.net/viewvc?view=revision&revision=301164
Log: - Fixed ext/tokenizer to work with the new tokens (<, >, @, !, etc) Changed paths: U php/php-src/branches/LEMON/ext/tokenizer/tokenizer.c U php/php-src/branches/LEMON/ext/tokenizer/tokenizer_data.c
Modified: php/php-src/branches/LEMON/ext/tokenizer/tokenizer.c =================================================================== --- php/php-src/branches/LEMON/ext/tokenizer/tokenizer.c 2010-07-11 13:21:00 UTC (rev 301163) +++ php/php-src/branches/LEMON/ext/tokenizer/tokenizer.c 2010-07-11 14:17:26 UTC (rev 301164) @@ -126,25 +126,55 @@ break; } - if (token_type >= 256) { - MAKE_STD_ZVAL(keyword); - array_init(keyword); - add_next_index_long(keyword, token_type); - if (token_type == T_END_HEREDOC) { - if (CG(increment_lineno)) { - token_line = ++CG(zend_lineno); - CG(increment_lineno) = 0; + switch (token_type) { + case T_DOT: + case T_AT: + case T_BOOL_NOT: + case T_BW_AND: + case T_BW_OR: + case T_BW_XOR: + case T_BW_NOT: + case T_PLUS: + case T_MINUS: + case T_MOD: + case T_DIV: + case T_MULT: + case T_SEMICOLON: + case T_COLON: + case T_COMMA: + case T_LBRACE: + case T_RBRACE: + case T_LBRACKET: + case T_RBRACKET: + case T_LPAREN: + case T_RPAREN: + case T_EQUAL: + case T_QUOTE: + case T_BACKQUOTE: + case T_QUESTION_MARK: + case T_IS_GREATER: + case T_IS_SMALLER: + add_next_index_stringl(return_value, (char *)zendtext, zendleng, 1); + break; + default: + MAKE_STD_ZVAL(keyword); + array_init(keyword); + add_next_index_long(keyword, token_type); + if (token_type == T_END_HEREDOC) { + if (CG(increment_lineno)) { + token_line = ++CG(zend_lineno); + CG(increment_lineno) = 0; + } + add_next_index_stringl(keyword, Z_STRVAL(token), Z_STRLEN(token), 1); + efree(Z_STRVAL(token)); + } else { + add_next_index_stringl(keyword, (char *)zendtext, zendleng, 1); } - add_next_index_stringl(keyword, Z_STRVAL(token), Z_STRLEN(token), 1); - efree(Z_STRVAL(token)); - } else { - add_next_index_stringl(keyword, (char *)zendtext, zendleng, 1); - } - add_next_index_long(keyword, token_line); - add_next_index_zval(return_value, keyword); - } else { - add_next_index_stringl(return_value, (char *)zendtext, zendleng, 1); + add_next_index_long(keyword, token_line); + add_next_index_zval(return_value, keyword); + break; } + if (destroy && Z_TYPE(token) != IS_NULL) { zval_dtor(&token); } Modified: php/php-src/branches/LEMON/ext/tokenizer/tokenizer_data.c =================================================================== --- php/php-src/branches/LEMON/ext/tokenizer/tokenizer_data.c 2010-07-11 13:21:00 UTC (rev 301163) +++ php/php-src/branches/LEMON/ext/tokenizer/tokenizer_data.c 2010-07-11 14:17:26 UTC (rev 301164) @@ -103,41 +103,55 @@ REGISTER_LONG_CONSTANT("T_PRIVATE", T_PRIVATE, CONST_CS | CONST_PERSISTENT); REGISTER_LONG_CONSTANT("T_PROTECTED", T_PROTECTED, CONST_CS | CONST_PERSISTENT); REGISTER_LONG_CONSTANT("T_PUBLIC", T_PUBLIC, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_OPEN_TAG", T_OPEN_TAG, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_OPEN_TAG_WITH_ECHO", T_OPEN_TAG_WITH_ECHO, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_WHITESPACE", T_WHITESPACE, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_COMMENT", T_COMMENT, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_CLOSE_TAG", T_CLOSE_TAG, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_DOC_COMMENT", T_DOC_COMMENT, CONST_CS | CONST_PERSISTENT); REGISTER_LONG_CONSTANT("T_STRING", T_STRING, CONST_CS | CONST_PERSISTENT); REGISTER_LONG_CONSTANT("T_NS_SEPARATOR", T_NS_SEPARATOR, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_NAMESPACE", T_NAMESPACE, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_LBRACE", T_LBRACE, CONST_CS | CONST_PERSISTENT); REGISTER_LONG_CONSTANT("T_HALT_COMPILER", T_HALT_COMPILER, CONST_CS | CONST_PERSISTENT); REGISTER_LONG_CONSTANT("T_LPAREN", T_LPAREN, CONST_CS | CONST_PERSISTENT); REGISTER_LONG_CONSTANT("T_RPAREN", T_RPAREN, CONST_CS | CONST_PERSISTENT); REGISTER_LONG_CONSTANT("T_SEMICOLON", T_SEMICOLON, CONST_CS | CONST_PERSISTENT); - REGISTER_LONG_CONSTANT("T_NAMESPACE", T_NAMESPACE, CONST_CS | CONST_PERSISTENT); - REGISTER_LONG_CONSTANT("T_LBRACE", T_LBRACE, CONST_CS | CONST_PERSISTENT); REGISTER_LONG_CONSTANT("T_RBRACE", T_RBRACE, CONST_CS | CONST_PERSISTENT); REGISTER_LONG_CONSTANT("T_USE", T_USE, CONST_CS | CONST_PERSISTENT); REGISTER_LONG_CONSTANT("T_AS", T_AS, CONST_CS | CONST_PERSISTENT); REGISTER_LONG_CONSTANT("T_CONST", T_CONST, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_ENDWHILE", T_ENDWHILE, CONST_CS | CONST_PERSISTENT); REGISTER_LONG_CONSTANT("T_WHILE", T_WHILE, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_DO", T_DO, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_ENDFOR", T_ENDFOR, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_FOR", T_FOR, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_ENDSWITCH", T_ENDSWITCH, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_CASE", T_CASE, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_DEFAULT", T_DEFAULT, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_SWITCH", T_SWITCH, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_ENDDECLARE", T_ENDDECLARE, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_DECLARE", T_DECLARE, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_DOUBLE_ARROW", T_DOUBLE_ARROW, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_ENDFOREACH", T_ENDFOREACH, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_FOREACH", T_FOREACH, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_CATCH", T_CATCH, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_VARIABLE", T_VARIABLE, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_TRY", T_TRY, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_GLOBAL", T_GLOBAL, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_ECHO", T_ECHO, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_UNSET", T_UNSET, CONST_CS | CONST_PERSISTENT); REGISTER_LONG_CONSTANT("T_BREAK", T_BREAK, CONST_CS | CONST_PERSISTENT); REGISTER_LONG_CONSTANT("T_CONTINUE", T_CONTINUE, CONST_CS | CONST_PERSISTENT); REGISTER_LONG_CONSTANT("T_RETURN", T_RETURN, CONST_CS | CONST_PERSISTENT); - REGISTER_LONG_CONSTANT("T_GLOBAL", T_GLOBAL, CONST_CS | CONST_PERSISTENT); - REGISTER_LONG_CONSTANT("T_ECHO", T_ECHO, CONST_CS | CONST_PERSISTENT); REGISTER_LONG_CONSTANT("T_INLINE_HTML", T_INLINE_HTML, CONST_CS | CONST_PERSISTENT); - REGISTER_LONG_CONSTANT("T_UNSET", T_UNSET, CONST_CS | CONST_PERSISTENT); - REGISTER_LONG_CONSTANT("T_FOREACH", T_FOREACH, CONST_CS | CONST_PERSISTENT); REGISTER_LONG_CONSTANT("T_THROW", T_THROW, CONST_CS | CONST_PERSISTENT); REGISTER_LONG_CONSTANT("T_GOTO", T_GOTO, CONST_CS | CONST_PERSISTENT); - REGISTER_LONG_CONSTANT("T_CATCH", T_CATCH, CONST_CS | CONST_PERSISTENT); - REGISTER_LONG_CONSTANT("T_VARIABLE", T_VARIABLE, CONST_CS | CONST_PERSISTENT); REGISTER_LONG_CONSTANT("T_CLASS", T_CLASS, CONST_CS | CONST_PERSISTENT); REGISTER_LONG_CONSTANT("T_TRAIT", T_TRAIT, CONST_CS | CONST_PERSISTENT); REGISTER_LONG_CONSTANT("T_EXTENDS", T_EXTENDS, CONST_CS | CONST_PERSISTENT); REGISTER_LONG_CONSTANT("T_INTERFACE", T_INTERFACE, CONST_CS | CONST_PERSISTENT); REGISTER_LONG_CONSTANT("T_IMPLEMENTS", T_IMPLEMENTS, CONST_CS | CONST_PERSISTENT); - REGISTER_LONG_CONSTANT("T_DOUBLE_ARROW", T_DOUBLE_ARROW, CONST_CS | CONST_PERSISTENT); - REGISTER_LONG_CONSTANT("T_ENDFOR", T_ENDFOR, CONST_CS | CONST_PERSISTENT); - REGISTER_LONG_CONSTANT("T_ENDFOREACH", T_ENDFOREACH, CONST_CS | CONST_PERSISTENT); - REGISTER_LONG_CONSTANT("T_ENDDECLARE", T_ENDDECLARE, CONST_CS | CONST_PERSISTENT); - REGISTER_LONG_CONSTANT("T_ENDWHILE", T_ENDWHILE, CONST_CS | CONST_PERSISTENT); REGISTER_LONG_CONSTANT("T_ARRAY", T_ARRAY, CONST_CS | CONST_PERSISTENT); REGISTER_LONG_CONSTANT("T_BOOL_HINT", T_BOOL_HINT, CONST_CS | CONST_PERSISTENT); REGISTER_LONG_CONSTANT("T_STRING_HINT", T_STRING_HINT, CONST_CS | CONST_PERSISTENT); @@ -151,6 +165,7 @@ REGISTER_LONG_CONSTANT("T_INSTEADOF", T_INSTEADOF, CONST_CS | CONST_PERSISTENT); REGISTER_LONG_CONSTANT("T_PAAMAYIM_NEKUDOTAYIM", T_PAAMAYIM_NEKUDOTAYIM, CONST_CS | CONST_PERSISTENT); REGISTER_LONG_CONSTANT("T_VAR", T_VAR, CONST_CS | CONST_PERSISTENT); + REGISTER_LONG_CONSTANT("T_LIST", T_LIST, CONST_CS | CONST_PERSISTENT); REGISTER_LONG_CONSTANT("T_BACKQUOTE", T_BACKQUOTE, CONST_CS | CONST_PERSISTENT); REGISTER_LONG_CONSTANT("T_FUNCTION", T_FUNCTION, CONST_CS | CONST_PERSISTENT); REGISTER_LONG_CONSTANT("T_OBJECT_OPERATOR", T_OBJECT_OPERATOR, CONST_CS | CONST_PERSISTENT); @@ -170,26 +185,11 @@ REGISTER_LONG_CONSTANT("T_STRING_VARNAME", T_STRING_VARNAME, CONST_CS | CONST_PERSISTENT); REGISTER_LONG_CONSTANT("T_QUOTE", T_QUOTE, CONST_CS | CONST_PERSISTENT); REGISTER_LONG_CONSTANT("T_LBRACKET", T_LBRACKET, CONST_CS | CONST_PERSISTENT); - REGISTER_LONG_CONSTANT("T_LIST", T_LIST, CONST_CS | CONST_PERSISTENT); REGISTER_LONG_CONSTANT("T_DOLLAR_OPEN_CURLY_BRACES", T_DOLLAR_OPEN_CURLY_BRACES, CONST_CS | CONST_PERSISTENT); REGISTER_LONG_CONSTANT("T_CURLY_OPEN", T_CURLY_OPEN, CONST_CS | CONST_PERSISTENT); REGISTER_LONG_CONSTANT("T_NUM_STRING", T_NUM_STRING, CONST_CS | CONST_PERSISTENT); REGISTER_LONG_CONSTANT("T_ISSET", T_ISSET, CONST_CS | CONST_PERSISTENT); REGISTER_LONG_CONSTANT("T_EMPTY", T_EMPTY, CONST_CS | CONST_PERSISTENT); - REGISTER_LONG_CONSTANT("T_OPEN_TAG", T_OPEN_TAG, CONST_CS | CONST_PERSISTENT); - REGISTER_LONG_CONSTANT("T_OPEN_TAG_WITH_ECHO", T_OPEN_TAG_WITH_ECHO, CONST_CS | CONST_PERSISTENT); - REGISTER_LONG_CONSTANT("T_WHITESPACE", T_WHITESPACE, CONST_CS | CONST_PERSISTENT); - REGISTER_LONG_CONSTANT("T_COMMENT", T_COMMENT, CONST_CS | CONST_PERSISTENT); - REGISTER_LONG_CONSTANT("T_CLOSE_TAG", T_CLOSE_TAG, CONST_CS | CONST_PERSISTENT); - REGISTER_LONG_CONSTANT("T_DOC_COMMENT", T_DOC_COMMENT, CONST_CS | CONST_PERSISTENT); - REGISTER_LONG_CONSTANT("T_DEFAULT", T_DEFAULT, CONST_CS | CONST_PERSISTENT); - REGISTER_LONG_CONSTANT("T_DECLARE", T_DECLARE, CONST_CS | CONST_PERSISTENT); - REGISTER_LONG_CONSTANT("T_DO", T_DO, CONST_CS | CONST_PERSISTENT); - REGISTER_LONG_CONSTANT("T_FOR", T_FOR, CONST_CS | CONST_PERSISTENT); - REGISTER_LONG_CONSTANT("T_CASE", T_CASE, CONST_CS | CONST_PERSISTENT); - REGISTER_LONG_CONSTANT("T_ENDSWITCH", T_ENDSWITCH, CONST_CS | CONST_PERSISTENT); - REGISTER_LONG_CONSTANT("T_TRY", T_TRY, CONST_CS | CONST_PERSISTENT); - REGISTER_LONG_CONSTANT("T_SWITCH", T_SWITCH, CONST_CS | CONST_PERSISTENT); REGISTER_LONG_CONSTANT("T_DOUBLE_COLON", T_PAAMAYIM_NEKUDOTAYIM, CONST_CS | CONST_PERSISTENT); } @@ -271,41 +271,55 @@ case T_PRIVATE: return "T_PRIVATE"; case T_PROTECTED: return "T_PROTECTED"; case T_PUBLIC: return "T_PUBLIC"; + case T_OPEN_TAG: return "T_OPEN_TAG"; + case T_OPEN_TAG_WITH_ECHO: return "T_OPEN_TAG_WITH_ECHO"; + case T_WHITESPACE: return "T_WHITESPACE"; + case T_COMMENT: return "T_COMMENT"; + case T_CLOSE_TAG: return "T_CLOSE_TAG"; + case T_DOC_COMMENT: return "T_DOC_COMMENT"; case T_STRING: return "T_STRING"; case T_NS_SEPARATOR: return "T_NS_SEPARATOR"; + case T_NAMESPACE: return "T_NAMESPACE"; + case T_LBRACE: return "T_LBRACE"; case T_HALT_COMPILER: return "T_HALT_COMPILER"; case T_LPAREN: return "T_LPAREN"; case T_RPAREN: return "T_RPAREN"; case T_SEMICOLON: return "T_SEMICOLON"; - case T_NAMESPACE: return "T_NAMESPACE"; - case T_LBRACE: return "T_LBRACE"; case T_RBRACE: return "T_RBRACE"; case T_USE: return "T_USE"; case T_AS: return "T_AS"; case T_CONST: return "T_CONST"; + case T_ENDWHILE: return "T_ENDWHILE"; case T_WHILE: return "T_WHILE"; + case T_DO: return "T_DO"; + case T_ENDFOR: return "T_ENDFOR"; + case T_FOR: return "T_FOR"; + case T_ENDSWITCH: return "T_ENDSWITCH"; + case T_CASE: return "T_CASE"; + case T_DEFAULT: return "T_DEFAULT"; + case T_SWITCH: return "T_SWITCH"; + case T_ENDDECLARE: return "T_ENDDECLARE"; + case T_DECLARE: return "T_DECLARE"; + case T_DOUBLE_ARROW: return "T_DOUBLE_ARROW"; + case T_ENDFOREACH: return "T_ENDFOREACH"; + case T_FOREACH: return "T_FOREACH"; + case T_CATCH: return "T_CATCH"; + case T_VARIABLE: return "T_VARIABLE"; + case T_TRY: return "T_TRY"; + case T_GLOBAL: return "T_GLOBAL"; + case T_ECHO: return "T_ECHO"; + case T_UNSET: return "T_UNSET"; case T_BREAK: return "T_BREAK"; case T_CONTINUE: return "T_CONTINUE"; case T_RETURN: return "T_RETURN"; - case T_GLOBAL: return "T_GLOBAL"; - case T_ECHO: return "T_ECHO"; case T_INLINE_HTML: return "T_INLINE_HTML"; - case T_UNSET: return "T_UNSET"; - case T_FOREACH: return "T_FOREACH"; case T_THROW: return "T_THROW"; case T_GOTO: return "T_GOTO"; - case T_CATCH: return "T_CATCH"; - case T_VARIABLE: return "T_VARIABLE"; case T_CLASS: return "T_CLASS"; case T_TRAIT: return "T_TRAIT"; case T_EXTENDS: return "T_EXTENDS"; case T_INTERFACE: return "T_INTERFACE"; case T_IMPLEMENTS: return "T_IMPLEMENTS"; - case T_DOUBLE_ARROW: return "T_DOUBLE_ARROW"; - case T_ENDFOR: return "T_ENDFOR"; - case T_ENDFOREACH: return "T_ENDFOREACH"; - case T_ENDDECLARE: return "T_ENDDECLARE"; - case T_ENDWHILE: return "T_ENDWHILE"; case T_ARRAY: return "T_ARRAY"; case T_BOOL_HINT: return "T_BOOL_HINT"; case T_STRING_HINT: return "T_STRING_HINT"; @@ -319,6 +333,7 @@ case T_INSTEADOF: return "T_INSTEADOF"; case T_PAAMAYIM_NEKUDOTAYIM: return "T_DOUBLE_COLON"; case T_VAR: return "T_VAR"; + case T_LIST: return "T_LIST"; case T_BACKQUOTE: return "T_BACKQUOTE"; case T_FUNCTION: return "T_FUNCTION"; case T_OBJECT_OPERATOR: return "T_OBJECT_OPERATOR"; @@ -338,26 +353,11 @@ case T_STRING_VARNAME: return "T_STRING_VARNAME"; case T_QUOTE: return "T_QUOTE"; case T_LBRACKET: return "T_LBRACKET"; - case T_LIST: return "T_LIST"; case T_DOLLAR_OPEN_CURLY_BRACES: return "T_DOLLAR_OPEN_CURLY_BRACES"; case T_CURLY_OPEN: return "T_CURLY_OPEN"; case T_NUM_STRING: return "T_NUM_STRING"; case T_ISSET: return "T_ISSET"; case T_EMPTY: return "T_EMPTY"; - case T_OPEN_TAG: return "T_OPEN_TAG"; - case T_OPEN_TAG_WITH_ECHO: return "T_OPEN_TAG_WITH_ECHO"; - case T_WHITESPACE: return "T_WHITESPACE"; - case T_COMMENT: return "T_COMMENT"; - case T_CLOSE_TAG: return "T_CLOSE_TAG"; - case T_DOC_COMMENT: return "T_DOC_COMMENT"; - case T_DEFAULT: return "T_DEFAULT"; - case T_DECLARE: return "T_DECLARE"; - case T_DO: return "T_DO"; - case T_FOR: return "T_FOR"; - case T_CASE: return "T_CASE"; - case T_ENDSWITCH: return "T_ENDSWITCH"; - case T_TRY: return "T_TRY"; - case T_SWITCH: return "T_SWITCH"; } return "UNKNOWN";
-- PHP CVS Mailing List (http://www.php.net/) To unsubscribe, visit: http://www.php.net/unsub.php