branch: externals/phpinspect commit daf9a6ca43851aa01fb81f97279df626a05f8030 Author: Hugo Thunnissen <de...@hugot.nl> Commit: Hugo Thunnissen <de...@hugot.nl>
WIP fixing incremental parser --- phpinspect-changeset.el | 20 ++++++------- phpinspect-edtrack.el | 24 ++++++++-------- phpinspect-parser.el | 30 ++++++++++++++------ test/test-buffer.el | 4 +++ test/test-changeset.el | 18 ++++++++++-- test/test-parser.el | 74 +++++++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 138 insertions(+), 32 deletions(-) diff --git a/phpinspect-changeset.el b/phpinspect-changeset.el index 7c375f772e..f75b41bf94 100644 --- a/phpinspect-changeset.el +++ b/phpinspect-changeset.el @@ -54,26 +54,26 @@ (inline-letevals (changeset) (inline-quote (progn + + (setf (phpinspect-meta-absolute-start (phpinspect-changeset-meta ,changeset)) + (phpinspect-changeset-start ,changeset)) + (setf (phpinspect-meta-absolute-end (phpinspect-changeset-meta ,changeset)) + (phpinspect-changeset-end ,changeset)) + (setf (phpinspect-meta-parent-offset (phpinspect-changeset-meta ,changeset)) + (phpinspect-changeset-parent-offset ,changeset)) + (let ((parent (phpinspect-changeset-parent ,changeset))) (unless (eq parent (phpinspect-meta-parent (phpinspect-changeset-meta ,changeset))) - (setf (phpinspect-meta-parent (phpinspect-changeset-meta ,changeset)) parent) (when parent (phpinspect-splayt-insert (phpinspect-meta-children parent) - (phpinspect-changeset-start ,changeset) - (phpinspect-changeset-meta ,changeset))))) - - (setf (phpinspect-meta-absolute-start (phpinspect-changeset-meta ,changeset)) - (phpinspect-changeset-start ,changeset)) - (setf (phpinspect-meta-absolute-end (phpinspect-changeset-meta ,changeset)) - (phpinspect-changeset-end ,changeset)) - (setf (phpinspect-meta-parent-offset (phpinspect-changeset-meta ,changeset)) - (phpinspect-changeset-parent-offset ,changeset)))))) + (phpinspect-changeset-parent-offset ,changeset) + (phpinspect-changeset-meta ,changeset))))))))) (provide 'phpinspect-changeset) ;;; phpinspect-changeset.el ends here diff --git a/phpinspect-edtrack.el b/phpinspect-edtrack.el index 45a11d68fd..d04ad0ecac 100644 --- a/phpinspect-edtrack.el +++ b/phpinspect-edtrack.el @@ -103,17 +103,19 @@ of all known edits before EDIT's start." point))) (defsubst phpinspect-edtrack-current-position-at-point (track point) - (let ((edit (phpinspect-edtrack-edits track)) - (encroached) - (pos)) + "Like `phpinspect-edtrack-original-position-at-point' but in reverse. + +Calculate the current position in of original POINT, based on edits performed." + (let ((edit (phpinspect-edtrack-edits track))) + ;; Find last edit before point (while (and edit (<= point (phpinspect-edit-original-end edit))) (setq edit (cdr edit))) - (setq pos (+ point (phpinspect-edit-delta edit))) - - (if (< 0 (setq encroached (- (+ (phpinspect-edit-original-end edit) (or (cdar edit) 0)) point))) - (- pos encroached) - pos))) + (if edit + ;; Subtract cumulative edit delta + (+ point (phpinspect-edit-delta edit)) + ;; else: no edits have taken place, return POINT + point))) (define-inline phpinspect-taint-start (taint) (inline-quote (car ,taint))) @@ -254,9 +256,9 @@ of all known edits before EDIT's start." (or (and (phpinspect-taint-iterator-follow ,iter (phpinspect-meta-start ,meta)) (phpinspect-taint-overlaps-meta (phpinspect-taint-iterator-current ,iter) ,meta)) - ;; Incomplete tokens should be regarded as tainted regardless of - ;; region, as their end-position is unlikely to be correct. - (phpinspect-incomplete-token-p (phpinspect-meta-token ,meta)))))) + ;; Incomplete tokens should be regarded as tainted regardless of + ;; region, as their end-position is unlikely to be correct. + (phpinspect-incomplete-token-p (phpinspect-meta-token ,meta)))))) (define-inline phpinspect-taint-iterator-region-is-tainted-p (iter start end) (inline-letevals (iter start end) diff --git a/phpinspect-parser.el b/phpinspect-parser.el index 0c30ba8445..59e35c697a 100644 --- a/phpinspect-parser.el +++ b/phpinspect-parser.el @@ -139,7 +139,7 @@ text at point and returns the resulting token." (defun phpinspect--recycle-token (context taint-iterator point original-point token-meta tokens-rear &optional delimiter-predicate) "Attempt to re-use TOKEN-META and any of its eligible righthand siblings." (declare (speed 3)) - + ;;(message "Recycle token called at %d, %s" (point) (phpinspect-meta-token token-meta)) (when-let ((bmap (phpinspect-pctx-bmap context)) (first-iteration t)) (catch 'phpinspect--return @@ -148,11 +148,20 @@ text at point and returns the resulting token." ;; Set point-offset-base for more efficient execution of ;; `phpinspect-meta-start' and related functions. (dlet ((phpinspect-meta--point-offset-base original-point)) + ;; (while (and token-meta (phpinspect-taint-iterator-token-is-tainted-p taint-iterator token-meta)) + ;; (pp taint-iterator) + ;; (message "finding child of %s" (phpinspect-meta-string token-meta)) + ;; (setq token-meta (phpinspect-meta-find-child-starting-at token-meta original-point))) + (if (or (not token-meta) (phpinspect-taint-iterator-token-is-tainted-p taint-iterator token-meta)) - ;; If the first passed token is tainted. Return tainted symbol to - ;; signal failure to parser loop. Otherwise return re-used tokens. - (throw 'phpinspect--return (if first-iteration 'tainted tokens-rear)) + (progn + ;; If the first passed token is tainted. Return tainted symbol to + ;; signal failure to parser loop. Otherwise return re-used tokens. + (throw 'phpinspect--return + (if first-iteration + (if (not token-meta) nil 'tainted) + tokens-rear))) ;; Token is eligible for re-use. (let ((parent-offset (phpinspect-meta-parent-offset token-meta)) @@ -163,6 +172,7 @@ text at point and returns the resulting token." ;; Note: recycling the token will update its parent and ;; positions. Its properties should no longer be queried (aside ;; from its width, which always stays the same) + ;;(message "recycling %s" (phpinspect-meta-string token-meta)) (phpinspect-bmap-recycle bmap token-meta delta (phpinspect-pctx-consume-whitespace context)) @@ -180,6 +190,7 @@ text at point and returns the resulting token." ((not (phpinspect-taint-iterator-region-is-tainted-p taint-iterator current-end-position (+ delta (phpinspect-meta-start right-sibling)))))) (progn + ;;(message "using sibling %s" (phpinspect-meta-string right-sibling)) ;; There was a right sibling and it is eligible for ;; re-use. Set token-meta and "recurse". (setq first-iteration nil @@ -277,6 +288,8 @@ is able to reuse an already parsed tree." (setq start-position (point)) + ;;(message "Start: %d" start-position) + (cond ((and-let* ((edtrack) (previous-bmap) @@ -298,11 +311,11 @@ is able to reuse an already parsed tree." (when-let ((token-after (phpinspect-bmap-token-starting-after previous-bmap original-position)) (start (phpinspect-meta-start token-after)) - ((not (phpinspect-taint-iterator-region-is-tainted-p - taint-iterator original-position start))) - (current-start (+ start-position (- start original-position)))) - (goto-char current-start) + taint-iterator original-position (+ start (phpinspect-meta-width token-after))))) + ;;(current-start (+ start-position (- start original-position)))) + (current-start (phpinspect-edtrack-current-position-at-point edtrack start))) + ;; (message "YAAS: (%d,%d) '%s'" start current-start (buffer-substring current-start (point-max))) (phpinspect--recycle-token context taint-iterator current-start start token-after tokens-rear ,(if delimiter-predicate `(quote ,delimiter-predicate) 'nil))))) @@ -316,6 +329,7 @@ is able to reuse an already parsed tree." ;; Re-using tokens was a success, update tokens-rear ((setq tokens-rear result)))) + ;;(message "recycled") ;; Skip over whitespace after so that we don't do a full ;; run down all of the handlers during the next iteration diff --git a/test/test-buffer.el b/test/test-buffer.el index 617f41b22f..28551780a4 100644 --- a/test/test-buffer.el +++ b/test/test-buffer.el @@ -331,10 +331,14 @@ class AAA { (insert " ")) (setq switch (not switch)) + (phpinspect-parse-string (buffer-string)) (phpinspect-buffer-parse buffer 'no-interrupt) (let ((token (phpinspect-bmap-last-token-before-point (phpinspect-buffer-map buffer) (+ 68 delta)))) + ;; (message "Map:") + ;; (dolist (meta (mapcar #'phpinspect-meta-string (sort (phpinspect-meta-flatten (phpinspect-buffer-root-meta buffer)) #'phpinspect-meta-sort-start))) + ;; (message " - %s" meta)) (should token) (should (phpinspect-variable-p (phpinspect-meta-token token))) (should (string= "banana" (cadr (phpinspect-meta-token token)))) diff --git a/test/test-changeset.el b/test/test-changeset.el index e7df5afff0..8d21bc0c9d 100644 --- a/test/test-changeset.el +++ b/test/test-changeset.el @@ -33,11 +33,14 @@ (ert-deftest phpinspect-meta-with-changeset-revert-parent-relation () (let ((parent (phpinspect-make-meta nil 1 20 "" 'parent)) - (child (phpinspect-make-meta nil 2 5 "" 'child)) + (child (phpinspect-make-meta nil 4 8 "" 'child)) (pctx (phpinspect-make-pctx)) - (other-parent (phpinspect-make-meta nil 1 20 "" 'other-parent))) + (other-parent (phpinspect-make-meta nil 3 20 "" 'other-parent)) + parent-offset) (phpinspect-meta-set-parent child parent) + (setq parent-offset (phpinspect-meta-parent-offset child)) + (phpinspect-with-parse-context pctx (phpinspect-meta-with-changeset child @@ -50,7 +53,16 @@ (let ((children (phpinspect-splayt-to-list (phpinspect-meta-children parent)))) (should (length= children 1)) - (should (eq 'child (phpinspect-meta-token (car children))))))) + (should (eq 'child (phpinspect-meta-token (car children)))) + + (should (= parent-offset (phpinspect-meta-parent-offset child))) + (should (= 4 (phpinspect-meta-start child))) + (should (= 8 (phpinspect-meta-end child))) + (should (= 4 (phpinspect-meta-width child))) + + (should (eq child (phpinspect-splayt-find + (phpinspect-meta-children parent) + (phpinspect-meta-parent-offset child))))))) ;;; test-changeset.el ends here diff --git a/test/test-parser.el b/test/test-parser.el index 341212e213..260b2c0f82 100644 --- a/test/test-parser.el +++ b/test/test-parser.el @@ -28,6 +28,7 @@ (require 'phpinspect-test-env (expand-file-name "phpinspect-test-env.el" (file-name-directory (macroexp-file-name)))) +(require 'phpinspect) (ert-deftest phpinspect-parse-bmap () @@ -60,6 +61,79 @@ class TestClass { (should (phpinspect-list-p (phpinspect-meta-token parent))) (should (phpinspect-block-p (phpinspect-meta-token (phpinspect-meta-parent parent))))))) +(ert-deftest phpinspect-parse-string-incrementally-single-edit () + (let ((ctx (phpinspect-make-pctx :incremental t :bmap (phpinspect-make-bmap)))) + (phpinspect-with-parse-context ctx + (phpinspect-parse-string " : 'abc' ")) + + (let* ((bmap (phpinspect-pctx-bmap ctx)) + (children (thread-last (phpinspect-bmap-root-meta bmap) + (phpinspect-meta-children) + (phpinspect-splayt-to-list))) + (string-meta (car children))) + + (should (length= children 1)) + (should (phpinspect-string-p (phpinspect-meta-token string-meta))) + (should (= 5 (phpinspect-meta-width string-meta))) + (should (= 5 (phpinspect-meta-start string-meta))) + (should (= 10 (phpinspect-meta-end string-meta))) + + (let ((edit-tracker (phpinspect-make-edtrack))) + (phpinspect-edtrack-register-edit edit-tracker 1 1 1) + + (let ((prev-string-meta string-meta) + (ctx (phpinspect-make-pctx :incremental t + :previous-bmap bmap + :bmap (phpinspect-make-bmap) + :edtrack edit-tracker))) + (phpinspect-with-parse-context ctx + (phpinspect-parse-string " : 'abc' ")) + + (setq children (thread-last (phpinspect-bmap-root-meta (phpinspect-pctx-bmap ctx)) + (phpinspect-meta-children) + (phpinspect-splayt-to-list)) + string-meta (car children)) + + ;;(message "%s" (mapcar #'phpinspect-meta-string children)) + + (should (length= children 1)) + (should (eq prev-string-meta string-meta)) + (should (phpinspect-string-p (phpinspect-meta-token string-meta))) + (should (= 5 (phpinspect-meta-width string-meta))) + (should (= 4 (phpinspect-meta-start string-meta))) + (should (= 9 (phpinspect-meta-end string-meta)))))))) + +(ert-deftest phpinspect-parse-string-incrementally-muti-edit () + (let ((ctx (phpinspect-make-pctx :incremental t :bmap (phpinspect-make-bmap)))) + (phpinspect-with-parse-context ctx + (phpinspect-parse-string " : 'abc' ")) + + (let ((edit-tracker (phpinspect-make-edtrack))) + (phpinspect-edtrack-register-edit edit-tracker 1 1 1) + (phpinspect-edtrack-register-edit edit-tracker 2 5 0) + + (let ((prev-string-meta (phpinspect-meta-first-child (phpinspect-bmap-root-meta (phpinspect-pctx-bmap ctx)))) + (ctx (phpinspect-make-pctx :incremental t + :previous-bmap (phpinspect-pctx-bmap ctx) + :bmap (phpinspect-make-bmap) + :edtrack edit-tracker))) + (phpinspect-with-parse-context ctx + (phpinspect-parse-string " : 'abc' ")) + + (let* ((children (thread-last (phpinspect-bmap-root-meta (phpinspect-pctx-bmap ctx)) + (phpinspect-meta-children) + (phpinspect-splayt-to-list))) + (string-meta (car children))) + + + ;;(message "%s" (mapcar #'phpinspect-meta-string children)) + (should (length= children 1)) + (should (eq prev-string-meta string-meta)) + (should (phpinspect-string-p (phpinspect-meta-token string-meta))) + (should (= 5 (phpinspect-meta-width string-meta))) + (should (= 7 (phpinspect-meta-start string-meta))) + (should (= 12 (phpinspect-meta-end string-meta)))))))) + (ert-deftest phpinspect-parse-comma () (let* ((code "(,)") (ctx (phpinspect-make-pctx :incremental t :bmap (phpinspect-make-bmap)))