Re: [PHP-DEV] Revert Tokenizer behavior for 5.4

[Nikita Popov]

I just set up an PHP environment and wrote a proper patch (including
test changes) to make it collect the next three tokens. It's a git
patch and I'm not sure whether it's compatible with SVN patches. I
would love it if this would go into 5.4 before beta. I didn't know how
one could fetch the rest into T_INLINE_HTML, so I'm hoping on help
here from someone who actually knowns C there :)

Thanks, Nikita

From d6941fad8c0d91a2ffacb0c965b654d1711753b4 Mon Sep 17 00:00:00 2001
From: Nikita Popov <nikic@chicken.(none)>
Date: Tue, 13 Sep 2011 09:07:21 +0200
Subject: [PATCH] Make token_get_all return next three tokens after 
__halt_compiler

---
 ext/tokenizer/tests/bug54089.phpt                  |   10 ++++-
 ext/tokenizer/tests/token_get_all_variation16.phpt |   45 ++++++++++++++++++--
 ext/tokenizer/tokenizer.c                          |   10 ++++-
 3 files changed, 57 insertions(+), 8 deletions(-)

diff --git a/ext/tokenizer/tests/bug54089.phpt 
b/ext/tokenizer/tests/bug54089.phpt
index e1f6d79..2f40510 100644
--- a/ext/tokenizer/tests/bug54089.phpt
+++ b/ext/tokenizer/tests/bug54089.phpt
@@ -17,7 +17,7 @@ foreach ($tokens as $t)
 var_dump($code);
 ?>
 --EXPECTF--
-array(2) {
+array(5) {
   [0]=>
   array(3) {
     [0]=>
@@ -36,5 +36,11 @@ array(2) {
     [2]=>
     int(1)
   }
+  [2]=>
+  string(1) "("
+  [3]=>
+  string(1) ")"
+  [4]=>
+  string(1) ";"
 }
-string(21) "<?php __halt_compiler"
+string(24) "<?php __halt_compiler();"
diff --git a/ext/tokenizer/tests/token_get_all_variation16.phpt 
b/ext/tokenizer/tests/token_get_all_variation16.phpt
index 39550bd..db9bde1 100644
--- a/ext/tokenizer/tests/token_get_all_variation16.phpt
+++ b/ext/tokenizer/tests/token_get_all_variation16.phpt
@@ -47,17 +47,17 @@ else
 
 list($value1,$value2) = $c;
 if(empty($value1) && !isset($value1)) {
-  __halt_compiler();
+  myFunction();
 }
 ?>';
 $tokens =  token_get_all($source);
 var_dump($tokens);
 
-echo "Done"
+echo "Done";
 ?>
 --EXPECTF--
 *** Testing token_get_all() : with different function constructs ***
-array(135) {
+array(142) {
   [0]=>
   array(3) {
     [0]=>
@@ -958,9 +958,46 @@ array(135) {
     [0]=>
     int(%d)
     [1]=>
-    string(15) "__halt_compiler"
+    string(10) "myFunction"
+    [2]=>
+    int(26)
+  }
+  [135]=>
+  string(1) "("
+  [136]=>
+  string(1) ")"
+  [137]=>
+  string(1) ";"
+  [138]=>
+  array(3) {
+    [0]=>
+    int(%d)
+    [1]=>
+    string(1) "
+"
     [2]=>
     int(26)
   }
+  [139]=>
+  string(1) "}"
+  [140]=>
+  array(3) {
+    [0]=>
+    int(%d)
+    [1]=>
+    string(1) "
+"
+    [2]=>
+    int(27)
+  }
+  [141]=>
+  array(3) {
+    [0]=>
+    int(%d)
+    [1]=>
+    string(2) "?>"
+    [2]=>
+    int(28)
+  }
 }
 Done
diff --git a/ext/tokenizer/tokenizer.c b/ext/tokenizer/tokenizer.c
index c6c901f..d502f9a 100644
--- a/ext/tokenizer/tokenizer.c
+++ b/ext/tokenizer/tokenizer.c
@@ -106,6 +106,7 @@ static void tokenize(zval *return_value TSRMLS_DC)
        int token_type;
        zend_bool destroy;
        int token_line = 1;
+       int need_tokens = -1; // for __halt_compiler lexing. -1 = disabled
 
        array_init(return_value);
 
@@ -152,8 +153,13 @@ static void tokenize(zval *return_value TSRMLS_DC)
 
                token_line = CG(zend_lineno);
 
-               if (token_type == T_HALT_COMPILER) {
-                       break;
+               // after T_HALT_COMPILER collect the next three non-dropped 
tokens
+               if (need_tokens != -1) {
+                       if (destroy && --need_tokens == 0) {
+                               break;
+                       }
+               } else if (token_type == T_HALT_COMPILER) {
+                       need_tokens = 3;
                }
        }
 }
-- 
1.7.4.1


-- 
PHP Internals - PHP Runtime Development Mailing List
To unsubscribe, visit: http://www.php.net/unsub.php