branch: externals/phpinspect
commit 135263c5335bd1e314651c05575328909335425f
Author: Hugo Thunnissen <de...@hugot.nl>
Commit: Hugo Thunnissen <de...@hugot.nl>
Add tests for incremental parsing + fix parser bugs that came to light
---
phpinspect-edtrack.el | 46 +++++++++++++++-------------
phpinspect-parser.el | 73 ++++++++++++++++++++++++++++----------------
test/phpinspect-test.el | 1 +
test/test-buffer.el | 81 ++++++++++++++++++++++++++++++++++++++++++++++++-
test/test-edtrack.el | 13 ++++++++
5 files changed, 166 insertions(+), 48 deletions(-)
diff --git a/phpinspect-edtrack.el b/phpinspect-edtrack.el
index 73e56b42a6..3cf40f6aae 100644
--- a/phpinspect-edtrack.el
+++ b/phpinspect-edtrack.el
@@ -44,27 +44,31 @@
(cons (car (phpinspect-edtrack-taint-pool track))
(cl-copy-list (cdr (phpinspect-edtrack-taint-pool track)))))
-(gv-define-setter phpinspect-taint-iterator-current (current iter) `(setcar
,iter ,current))
-
-(defsubst phpinspect-taint-iterator-current (iter)
- (car iter))
-
-(defsubst phpinspect-taint-iterator-follow (iter pos)
- (or (while (and (phpinspect-taint-iterator-current iter)
- (> pos (phpinspect-taint-end
- (phpinspect-taint-iterator-current iter))))
- (setf (phpinspect-taint-iterator-current iter) (pop (cdr iter))))
- (phpinspect-taint-iterator-current iter)))
-
-(defsubst phpinspect-taint-iterator-token-is-tainted-p (iter meta)
- (and (phpinspect-taint-iterator-follow iter (phpinspect-meta-start meta))
- (phpinspect-taint-overlaps-meta
- (phpinspect-taint-iterator-current iter) meta)))
-
-(defsubst phpinspect-taint-iterator-region-is-tainted-p (iter start end)
- (and (phpinspect-taint-iterator-follow iter start)
- (phpinspect-taint-overlaps-region
- (phpinspect-taint-iterator-current iter) start end)))
+(define-inline phpinspect-taint-iterator-current (iter)
+ (inline-quote (car ,iter)))
+
+(define-inline phpinspect-taint-iterator-follow (iter pos)
+ (inline-letevals (iter pos)
+ (inline-quote
+ (or (while (and (phpinspect-taint-iterator-current ,iter)
+ (> ,pos (phpinspect-taint-end
+ (phpinspect-taint-iterator-current ,iter))))
+ (setf (phpinspect-taint-iterator-current ,iter) (pop (cdr ,iter))))
+ (phpinspect-taint-iterator-current ,iter)))))
+
+(define-inline phpinspect-taint-iterator-token-is-tainted-p (iter meta)
+ (inline-letevals (iter meta)
+ (inline-quote
+ (and (phpinspect-taint-iterator-follow ,iter (phpinspect-meta-start
,meta))
+ (phpinspect-taint-overlaps-meta
+ (phpinspect-taint-iterator-current ,iter) ,meta)))))
+
+(define-inline phpinspect-taint-iterator-region-is-tainted-p (iter start end)
+ (inline-letevals (iter start end)
+ (inline-quote
+ (and (phpinspect-taint-iterator-follow ,iter ,start)
+ (phpinspect-taint-overlaps-region
+ (phpinspect-taint-iterator-current ,iter) ,start ,end)))))
(defsubst phpinspect-edit-original-end (edit)
(or (caar edit) 0))
diff --git a/phpinspect-parser.el b/phpinspect-parser.el
index 2e76e3a40c..fc50cd0a3c 100644
--- a/phpinspect-parser.el
+++ b/phpinspect-parser.el
@@ -245,9 +245,13 @@ Type can be any of the token types returned by
(defsubst phpinspect-class-block (class)
(caddr class))
+(define-inline phpinspect-namespace-is-blocked-p (namespace)
+ (inline-letevals (namespace)
+ (inline-quote
+ (and (= (length ,namespace) 3) (phpinspect-block-p (caddr ,namespace))))))
+
(defsubst phpinspect-namespace-block (namespace)
- (when (and (= (length namespace) 3)
- (phpinspect-block-p (caddr namespace)))
+ (when (phpinspect-namespace-is-blocked-p namespace)
(caddr namespace)))
(defsubst phpinspect-function-block (php-func)
@@ -355,10 +359,11 @@ token is \";\", which marks the end of a statement in
PHP."
(let ((delimiter-predicate (if (symbolp delimiter-predicate)
`(quote ,delimiter-predicate)
delimiter-predicate)))
- `(defsubst ,(phpinspect-parser-func-name name "simple") (buffer max-point
&optional continue-condition &rest _ignored)
+ `(defsubst ,(phpinspect-parser-func-name name "simple") (buffer max-point
&optional skip-over continue-condition &rest _ignored)
(with-current-buffer buffer
(let (tokens token
- (delimiter-predicate (when (functionp ,delimiter-predicate)
,delimiter-predicate)))
+ (delimiter-predicate (when (functionp
,delimiter-predicate) ,delimiter-predicate)))
+ (when skip-over (forward-char skip-over))
(while (and (< (point) max-point)
(if continue-condition (funcall continue-condition) t)
(not (if delimiter-predicate
@@ -386,7 +391,7 @@ token is \";\", which marks the end of a statement in PHP."
(let ((delimiter-predicate (if (symbolp delimiter-predicate)
`(quote ,delimiter-predicate)
delimiter-predicate)))
- `(defsubst ,(phpinspect-parser-func-name name "incremental") (context
buffer max-point &optional continue-condition root)
+ `(defsubst ,(phpinspect-parser-func-name name "incremental") (context
buffer max-point &optional skip-over continue-condition root)
(with-current-buffer buffer
(let* ((tokens (list ,tree-type))
(root-start (point))
@@ -404,6 +409,7 @@ token is \";\", which marks the end of a statement in PHP."
(delta)
(token)
(delimiter-predicate (when (functionp ,delimiter-predicate)
,delimiter-predicate)))
+ (when skip-over (forward-char skip-over))
(phpinspect-pctx-save-whitespace context
(while (and (< (point) max-point)
(if continue-condition (funcall continue-condition) t)
@@ -426,6 +432,7 @@ token is \";\", which marks the end of a statement in PHP."
current-end-position (+ (phpinspect-meta-end
existing-meta) delta)
token (phpinspect-meta-token existing-meta))
+ ;;(message "Reusing token %s at point %s"
(phpinspect-meta-string existing-meta) (point))
;; Re-register existing token
(phpinspect-bmap-overlay
bmap previous-bmap existing-meta delta
@@ -510,12 +517,26 @@ executing.")
(let ((func-name (phpinspect-parser-func-name (phpinspect-parser-name
parser)))
(incremental-name (phpinspect-parser-func-name (phpinspect-parser-name
parser) "incremental"))
(simple-name (phpinspect-parser-func-name (phpinspect-parser-name
parser) "simple")))
- `(defun ,func-name (buffer max-point &optional continue-condition root)
+ `(defun ,func-name (buffer max-point &optional skip-over
continue-condition root)
+ "Parse BUFFER, starting at point and ending at MAX-POINT.
+
+If SKIP-OVER is non-nil, it must be a number of characters that
+to skip over before starting to parse.
+
+If CONTINUE-CONDITION is non-nil, it must be a function. It will
+be called after each parsed child token with the token as
+argument. If the return value is nil, parsing is stopped.
+
+If ROOT is non-nil, this signals that there is no parent parser
+that will take care of registering metadata for the parser's
+returned token tree. So the parser should register the metadata
+of the root of its returned tree itself, before
+returning. Currently, token metadata is only registered when
+parsing incrementally."
(if (and phpinspect-parse-context
(phpinspect-pctx-incremental phpinspect-parse-context))
- (,incremental-name phpinspect-parse-context buffer max-point
continue-condition root)
- (,simple-name buffer max-point continue-condition root)))))
-
+ (,incremental-name phpinspect-parse-context buffer max-point
skip-over continue-condition root)
+ (,simple-name buffer max-point skip-over continue-condition root)))))
(defmacro phpinspect-defparser (name &rest parameters)
(declare (indent 1))
@@ -677,12 +698,12 @@ executing.")
(doc-block (save-restriction
(goto-char region-start)
(narrow-to-region region-start region-end)
- (phpinspect--parse-doc-block (current-buffer)
(point-max) nil))))
+ (phpinspect--parse-doc-block (current-buffer)
(point-max)))))
(forward-char 2)
doc-block))
(t
(let ((end-position (line-end-position)))
- (phpinspect--parse-comment (current-buffer) end-position nil)))))
+ (phpinspect--parse-comment (current-buffer) end-position)))))
(phpinspect-defhandler variable (start-token &rest _ignored)
"Handler for tokens indicating reference to a variable"
@@ -724,7 +745,7 @@ executing.")
((regexp . (concat "use" (phpinspect--word-end-regex))))
(setq start-token (phpinspect--strip-word-end-space start-token))
(forward-char (length start-token))
- (phpinspect--parse-use (current-buffer) max-point nil))
+ (phpinspect--parse-use (current-buffer) max-point))
(phpinspect-defhandler attribute-reference (start-token &rest _ignored)
"Handler for references to object attributes, or static class attributes."
@@ -757,6 +778,7 @@ executing.")
(phpinspect--parse-namespace
(current-buffer)
max-point
+ nil
(lambda () (not (looking-at (phpinspect-handler-regexp namespace))))))
(phpinspect-defparser const
@@ -770,7 +792,7 @@ executing.")
(setq start-token (phpinspect--strip-word-end-space start-token))
(forward-char (length start-token))
- (setq start-token (phpinspect--parse-const (current-buffer) max-point nil))
+ (setq start-token (phpinspect--parse-const (current-buffer) max-point))
(when (phpinspect-incomplete-token-p (car (last start-token)))
(setcar start-token :incomplete-const))
start-token)
@@ -790,13 +812,12 @@ executing.")
"Handler for code blocks that cannot contain scope, const or
static keywords with the same meaning as in a class block."
((regexp . "{"))
- (forward-char (length start-token))
(let* ((complete-block nil)
(continue-condition (lambda ()
(not (and (char-equal (char-after) ?})
(setq complete-block t)))))
(parsed (phpinspect--parse-block-without-scopes
- (current-buffer) max-point continue-condition 'root)))
+ (current-buffer) max-point (length start-token)
continue-condition 'root)))
(if complete-block
(forward-char)
(setcar parsed :incomplete-block))
@@ -818,7 +839,7 @@ static keywords with the same meaning as in a class block."
(not (and (char-equal (char-after) ?})
(setq complete-block t)))))
(parsed (phpinspect--parse-class-block
- (current-buffer) max-point continue-condition 'root)))
+ (current-buffer) max-point (length start-token)
continue-condition 'root)))
(if complete-block
(forward-char)
(setcar parsed :incomplete-block))
@@ -830,7 +851,6 @@ static keywords with the same meaning as in a class block."
(phpinspect-defhandler block (start-token max-point)
"Handler for code blocks"
((regexp . "{"))
- (forward-char (length start-token))
(let* ((complete-block nil)
(continue-condition (lambda ()
;; When we encounter a closing brace for this
@@ -838,7 +858,7 @@ static keywords with the same meaning as in a class block."
(not (and (char-equal (char-after) ?})
(setq complete-block t)))))
(parsed (phpinspect--parse-block
- (current-buffer) max-point continue-condition)))
+ (current-buffer) max-point (length start-token)
continue-condition)))
(if complete-block
;; After meeting the char-after requirement above, we need to move
;; one char forward to prevent parent-blocks from exiting because
@@ -893,11 +913,11 @@ Returns the consumed text string without face properties."
datatypes like arrays, merely lists that are of a syntactic
nature like argument lists"
((regexp . "("))
- (forward-char (length start-token))
(let* ((complete-list nil)
(php-list (phpinspect--parse-list
(current-buffer)
max-point
+ (length start-token)
(lambda () (not (and (char-equal (char-after) ?\)) (setq
complete-list t)))))))
(if complete-list
@@ -916,7 +936,7 @@ nature like argument lists"
;; don't necessarily require the same handlers to parse.
(define-inline phpinspect-parse-declaration (buffer max-point &optional
continue-condition root)
(inline-quote
- (let ((result (phpinspect--parse-declaration ,buffer ,max-point
,continue-condition ,root)))
+ (let ((result (phpinspect--parse-declaration ,buffer ,max-point nil
,continue-condition ,root)))
(if (phpinspect-terminator-p (car (last result)))
(butlast result)
result))))
@@ -992,13 +1012,13 @@ nature like argument lists"
(phpinspect-defhandler array (start-token max-point)
"Handler for arrays, in the bracketet as well as the list notation"
((regexp . "\\[\\|array("))
- (forward-char (length start-token))
(let* ((end-char (cond ((string= start-token "[") ?\])
((string= start-token "array(") ?\))))
(end-char-reached nil)
(token (phpinspect--parse-array
(current-buffer)
max-point
+ (length start-token)
(lambda () (not (and (char-equal (char-after) end-char)
(setq end-char-reached t)))))))
@@ -1014,15 +1034,16 @@ nature like argument lists"
"Handler for the class keyword, and tokens that follow to define
the properties of the class"
((regexp . (concat "\\(abstract\\|final\\|class\\|interface\\|trait\\)"
- (phpinspect--word-end-regex))))
+ (phpinspect--word-end-regex))))
(setq start-token (phpinspect--strip-word-end-space start-token))
- (list :class (phpinspect-parse-declaration
+ `(:class ,(phpinspect-parse-declaration
(current-buffer)
max-point
(lambda () (not (char-equal (char-after) ?{)))
'root)
- (phpinspect--class-block-handler
- (char-to-string (char-after)) max-point)))
+ ,@(when (looking-at (phpinspect--class-block-handler-regexp))
+ (list (phpinspect--class-block-handler
+ (char-to-string (char-after)) max-point)))))
(phpinspect-defparser root
:tree-keyword "root"
@@ -1037,7 +1058,7 @@ the properties of the class"
(save-excursion
(goto-char (point-min))
(re-search-forward "<\\?php\\|<\\?" nil t)
- (phpinspect--parse-root (current-buffer) point nil 'root))))
+ (phpinspect--parse-root (current-buffer) point nil nil 'root))))
(defun phpinspect-parse-current-buffer ()
(phpinspect-parse-buffer-until-point
diff --git a/test/phpinspect-test.el b/test/phpinspect-test.el
index 86ba1cfa2f..a4e50bb233 100644
--- a/test/phpinspect-test.el
+++ b/test/phpinspect-test.el
@@ -543,6 +543,7 @@ class Thing
(load-file (concat phpinspect-test-directory "/test-parser.el"))
(load-file (concat phpinspect-test-directory "/test-parse-context.el"))
(load-file (concat phpinspect-test-directory "/test-splayt.el"))
+(load-file (concat phpinspect-test-directory "/test-pipeline.el"))
(provide 'phpinspect-test)
diff --git a/test/test-buffer.el b/test/test-buffer.el
index 0f9064afae..4d1c31fb7b 100644
--- a/test/test-buffer.el
+++ b/test/test-buffer.el
@@ -63,7 +63,7 @@
(cl-defstruct (phpinspect-document (:constructor phpinspect-make-document))
(buffer (get-buffer-create
- (generate-new-buffer-name " **phpinspect-document** shadow
buffer") t)
+ (generate-new-buffer-name "**phpinspect-document** shadow
buffer") t)
:type buffer
:documentation
"A hidden buffer with a reference version of the document."))
@@ -81,6 +81,16 @@
(erase-buffer)
(insert contents)))
+(defmacro phpinspect-document-setq-local (document &rest assignments)
+ (declare (indent 1))
+ `(with-current-buffer (phpinspect-document-buffer ,document)
+ (setq-local ,@assignments)))
+
+(defmacro phpinspect-with-document-buffer (document &rest body)
+ (declare (indent 1))
+ `(with-current-buffer (phpinspect-document-buffer ,document)
+ ,@body))
+
(cl-defmethod phpinspect-document-contents ((document phpinspect-document))
(with-current-buffer (phpinspect-document-buffer document)
(buffer-string)))
@@ -206,3 +216,72 @@ class AccountStatisticsController {
(should class)
(should (= class-location (phpinspect-meta-start class)))
(should (phpinspect-class-p (phpinspect-meta-token class)))))))))
+
+
+(ert-deftest phpinspect-buffer-parse-incrementally-multiedit ()
+ (let* ((document (phpinspect-make-document))
+ (buffer (phpinspect-make-buffer
+ :buffer (phpinspect-document-buffer document)))
+ parsed parsed-after current-tree)
+
+ (phpinspect-document-set-contents
+ document
+ "<?php
+
+
+namespace XXX;
+
+use ZZZ\\zzz;
+
+
+
+
+
+class YYY {
+
+ function Foo() {
+ if(bar()) {
+ return $baz->bip->bop(bar($bim), $bom)
+ }
+ }
+}")
+
+ (phpinspect-document-setq-local document
+ phpinspect-current-buffer buffer)
+ (phpinspect-with-document-buffer document
+ (setq buffer-undo-list nil)
+ (add-hook 'after-change-functions #'phpinspect-after-change-function))
+
+ (setq parsed (phpinspect-buffer-parse buffer 'no-interrupt))
+
+ ;; Delete lines before class
+ (phpinspect-with-document-buffer document
+ (goto-char 40)
+ (kill-line)
+ (kill-line)
+ (kill-line))
+
+ (setq parsed-after (phpinspect-buffer-parse buffer 'no-interrupt))
+
+ (should (equal parsed parsed-after))
+
+ ;; Delete namespace declaration
+ (phpinspect-with-document-buffer document
+ (goto-char 9)
+ (kill-line))
+
+ (setq parsed-after (phpinspect-buffer-parse buffer 'no-interrupt))
+ (setq current-tree (phpinspect-with-document-buffer document
+ (goto-char (point-min))
+ (phpinspect-parse-buffer-until-point (current-buffer)
(point-max))))
+
+ (should (equal current-tree parsed-after))
+
+ ;;Bring back the namespace declaration
+ (phpinspect-with-document-buffer document
+ (undo-start)
+ (undo-more 1))
+
+ (setq parsed-after (phpinspect-buffer-parse buffer 'no-interrupt))
+
+ (should (equal parsed parsed-after))))
diff --git a/test/test-edtrack.el b/test/test-edtrack.el
index e08750c73c..3b150dd2a3 100644
--- a/test/test-edtrack.el
+++ b/test/test-edtrack.el
@@ -1,5 +1,6 @@
(require 'ert)
(require 'phpinspect-edtrack)
+(require 'phpinspect-meta)
(ert-deftest phpinspect-edit-end ()
(let ((edit (list (cons 10 3) (cons 6 5) (cons 4 -2))))
@@ -126,3 +127,15 @@
(phpinspect-edtrack-register-edit track 10 10 1)
(should (equal (list (cons 10 -1) (cons 10 1)) (phpinspect-edtrack-edits
track)))))
+
+(ert-deftest phpinspect-edtrack-undo ()
+ (let ((track (phpinspect-make-edtrack)))
+ (phpinspect-edtrack-register-edit track 10 10 10)
+ (phpinspect-edtrack-register-edit track 10 10 10)
+ (phpinspect-edtrack-register-edit track 10 30 0)
+
+ (should (= 30 (phpinspect-edtrack-original-position-at-point track 30)))
+ (should (= 20 (phpinspect-edtrack-original-position-at-point track 20)))
+ (should (= 15 (phpinspect-edtrack-original-position-at-point track 15)))
+ (should (= 35 (phpinspect-edtrack-original-position-at-point track 35)))
+ (should (= 10 (phpinspect-edtrack-original-position-at-point track 10)))))
