branch: elpa/haskell-tng-mode commit 3ca869f17f7ac0c32b814b3f1605a6447f1d3d68 Author: Tseen She <ts33n....@gmail.com> Commit: Tseen She <ts33n....@gmail.com>
code reorganisation --- haskell-tng-font-lock.el | 254 ++++++++++----------- haskell-tng-layout.el | 34 +-- haskell-tng-lexer.el | 157 +++++++++++++ haskell-tng-smie.el | 155 +------------ test/haskell-tng-layout-test.el | 2 +- ...-tng-smie-test.el => haskell-tng-lexer-test.el} | 182 +++++++-------- test/haskell-tng-smie-test.el | 155 +------------ test/haskell-tng-testutils.el | 8 +- 8 files changed, 410 insertions(+), 537 deletions(-) diff --git a/haskell-tng-font-lock.el b/haskell-tng-font-lock.el index f20c544..7fe6f56 100644 --- a/haskell-tng-font-lock.el +++ b/haskell-tng-font-lock.el @@ -104,107 +104,107 @@ ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;; Here is the `font-lock-keywords' table of matchers and highlighters. (defvar - haskell-tng:keywords - ;; These regexps use the `rx' library so we can reuse common subpatterns. It - ;; also increases the readability of the code and, in many cases, allows us to - ;; do more work in a single regexp instead of multiple passes. - (let ((conid haskell-tng:rx:conid) - (qual haskell-tng:rx:qual) - (consym haskell-tng:rx:consym) - (toplevel haskell-tng:rx:toplevel) - (bigspace `(| space ,haskell-tng:rx:newline))) - `(;; reservedid / reservedop - (,haskell-tng:regexp:reserved - . 'haskell-tng:keyword) - - ;; Some things are not technically keywords but are always special so make - ;; sense to be fontified as such. - (,(rx (any ?\( ?\) ?\[ ?\] ?\{ ?\} ?,)) - (0 'haskell-tng:keyword)) - - ;; TypeFamilies - (,(rx word-start "type" (+ space) (group "family") word-end) - (1 'haskell-tng:keyword)) - ;; EXT:TypeFamilies (associated types, is this the right extension?) - - ;; Types - (haskell-tng:font:explicit-type:keyword - (1 'haskell-tng:type keep)) - (haskell-tng:font:topdecl:keyword - (1 'haskell-tng:type keep)) - (haskell-tng:font:type:keyword - (1 'haskell-tng:type keep)) - (haskell-tng:font:deriving:keyword - (1 'haskell-tng:keyword keep) - (2 'haskell-tng:type keep)) - - ;; EXT:TypeApplications: It is not easy to disambiguate between type - ;; applications and value extractor in a pattern. Needs work. - ;; (,(rx-to-string `(: symbol-start "@" (* space) - ;; (group (opt ,qual) (| ,conid ,consym)))) - ;; (1 'haskell-tng:type)) - - ;; imports - (haskell-tng:font:import:keyword - (,(rx-to-string - `(: line-start "import" (+ space) - (group (opt word-start "qualified" word-end)) (* space) - ;; EXT:PackageImports - ;; EXT:Safe, EXT:Trustworthy, EXT:Unsafe - (group symbol-start (* ,conid ".") ,conid symbol-end) (* ,bigspace) - (group (opt word-start "hiding" word-end)) (* space))) - (haskell-tng:font:multiline:anchor-rewind) nil - (1 'haskell-tng:keyword) - (2 'haskell-tng:module) - (3 'haskell-tng:keyword)) - (,(rx-to-string `(: word-start (group "as") word-end (+ space) - word-start (group ,conid) word-end)) - (haskell-tng:font:multiline:anchor-rewind) nil - (1 'haskell-tng:keyword) - (2 'haskell-tng:module)) - (haskell-tng:font:explicit-constructors - (haskell-tng:font:multiline:anchor-rewind 1) - (haskell-tng:font:multiline:anchor-rewind) - (0 'haskell-tng:constructor keep)) - (,(rx-to-string `(: word-start ,conid word-end)) - (haskell-tng:font:multiline:anchor-rewind 1) - (haskell-tng:font:multiline:anchor-rewind) - (0 'haskell-tng:type keep)) - ;; EXT:ExplicitNamespaces - ) - - (haskell-tng:font:module:keyword - (,(rx-to-string `(: word-start "module" word-end (+ space) - (group symbol-start (* ,conid ".") ,conid symbol-end))) - (haskell-tng:font:multiline:anchor-rewind) - (haskell-tng:font:multiline:anchor-rewind) - (1 'haskell-tng:module)) - (haskell-tng:font:explicit-constructors - (haskell-tng:font:multiline:anchor-rewind 1) - (haskell-tng:font:multiline:anchor-rewind) - (0 'haskell-tng:constructor keep)) - (,(rx-to-string `(: word-start ,conid word-end)) - (haskell-tng:font:multiline:anchor-rewind 1) - (haskell-tng:font:multiline:anchor-rewind) - (0 'haskell-tng:type keep))) - - ;; TODO pragmas - ;; TODO numeric / char primitives? - ;; TODO haddock, different face vs line comments, and some markup. - - ;; top-level - (,(rx-to-string toplevel) - . 'haskell-tng:toplevel) - - ;; uses of F.Q.N.s - (,(rx-to-string `(: symbol-start (+ (: ,conid ".")))) - . 'haskell-tng:module) - - ;; constructors - (,(rx-to-string `(: symbol-start (| ,conid ,consym) symbol-end)) - . 'haskell-tng:constructor) - - ))) + haskell-tng:keywords + ;; These regexps use the `rx' library so we can reuse common subpatterns. It + ;; also increases the readability of the code and, in many cases, allows us to + ;; do more work in a single regexp instead of multiple passes. + (let ((conid haskell-tng:rx:conid) + (qual haskell-tng:rx:qual) + (consym haskell-tng:rx:consym) + (toplevel haskell-tng:rx:toplevel) + (bigspace `(| space ,haskell-tng:rx:newline))) + `(;; reservedid / reservedop + (,haskell-tng:regexp:reserved + . 'haskell-tng:keyword) + + ;; Some things are not technically keywords but are always special so make + ;; sense to be fontified as such. + (,(rx (any ?\( ?\) ?\[ ?\] ?\{ ?\} ?,)) + (0 'haskell-tng:keyword)) + + ;; TypeFamilies + (,(rx word-start "type" (+ space) (group "family") word-end) + (1 'haskell-tng:keyword)) + ;; EXT:TypeFamilies (associated types, is this the right extension?) + + ;; Types + (haskell-tng:font:explicit-type:keyword + (1 'haskell-tng:type keep)) + (haskell-tng:font:topdecl:keyword + (1 'haskell-tng:type keep)) + (haskell-tng:font:type:keyword + (1 'haskell-tng:type keep)) + (haskell-tng:font:deriving:keyword + (1 'haskell-tng:keyword keep) + (2 'haskell-tng:type keep)) + + ;; EXT:TypeApplications: It is not easy to disambiguate between type + ;; applications and value extractor in a pattern. Needs work. + ;; (,(rx-to-string `(: symbol-start "@" (* space) + ;; (group (opt ,qual) (| ,conid ,consym)))) + ;; (1 'haskell-tng:type)) + + ;; imports + (haskell-tng:font:import:keyword + (,(rx-to-string + `(: line-start "import" (+ space) + (group (opt word-start "qualified" word-end)) (* space) + ;; EXT:PackageImports + ;; EXT:Safe, EXT:Trustworthy, EXT:Unsafe + (group symbol-start (* ,conid ".") ,conid symbol-end) (* ,bigspace) + (group (opt word-start "hiding" word-end)) (* space))) + (haskell-tng:font:multiline:anchor-rewind) nil + (1 'haskell-tng:keyword) + (2 'haskell-tng:module) + (3 'haskell-tng:keyword)) + (,(rx-to-string `(: word-start (group "as") word-end (+ space) + word-start (group ,conid) word-end)) + (haskell-tng:font:multiline:anchor-rewind) nil + (1 'haskell-tng:keyword) + (2 'haskell-tng:module)) + (haskell-tng:font:explicit-constructors + (haskell-tng:font:multiline:anchor-rewind 1) + (haskell-tng:font:multiline:anchor-rewind) + (0 'haskell-tng:constructor keep)) + (,(rx-to-string `(: word-start ,conid word-end)) + (haskell-tng:font:multiline:anchor-rewind 1) + (haskell-tng:font:multiline:anchor-rewind) + (0 'haskell-tng:type keep)) + ;; EXT:ExplicitNamespaces + ) + + (haskell-tng:font:module:keyword + (,(rx-to-string `(: word-start "module" word-end (+ space) + (group symbol-start (* ,conid ".") ,conid symbol-end))) + (haskell-tng:font:multiline:anchor-rewind) + (haskell-tng:font:multiline:anchor-rewind) + (1 'haskell-tng:module)) + (haskell-tng:font:explicit-constructors + (haskell-tng:font:multiline:anchor-rewind 1) + (haskell-tng:font:multiline:anchor-rewind) + (0 'haskell-tng:constructor keep)) + (,(rx-to-string `(: word-start ,conid word-end)) + (haskell-tng:font:multiline:anchor-rewind 1) + (haskell-tng:font:multiline:anchor-rewind) + (0 'haskell-tng:type keep))) + + ;; TODO pragmas + ;; TODO numeric / char primitives? + ;; TODO haddock, different face vs line comments, and some markup. + + ;; top-level + (,(rx-to-string toplevel) + . 'haskell-tng:toplevel) + + ;; uses of F.Q.N.s + (,(rx-to-string `(: symbol-start (+ (: ,conid ".")))) + . 'haskell-tng:module) + + ;; constructors + (,(rx-to-string `(: symbol-start (| ,conid ,consym) symbol-end)) + . 'haskell-tng:constructor) + + ))) (defun haskell-tng:font:multiline:anchor-rewind (&optional group jump) "MATCH-ANCHORED moving point to group beginning (plus JUMP) and declaring LIMIT. @@ -303,22 +303,22 @@ succeeds and may further restrict the FIND search limit." (add-to-list 'haskell-tng:extend-region-functions ',extend t))))) (haskell-tng:font:multiline explicit-type - (rx symbol-start "::" symbol-end) - (rx symbol-start "::" symbol-end (group (+ anything))) - haskell-tng:paren-close - haskell-tng:indent-close-previous) + (rx symbol-start "::" symbol-end) + (rx symbol-start "::" symbol-end (group (+ anything))) + haskell-tng:paren-close + haskell-tng:indent-close-previous) (haskell-tng:font:multiline topdecl - (rx line-start (| "data" "newtype" "class" "instance") word-end) - (rx line-start (| "data" "newtype" "class" "instance") word-end - (group (+? anything)) - (| (: line-start symbol-start) - (: symbol-start (| "where" "=") symbol-end)))) + (rx line-start (| "data" "newtype" "class" "instance") word-end) + (rx line-start (| "data" "newtype" "class" "instance") word-end + (group (+? anything)) + (| (: line-start symbol-start) + (: symbol-start (| "where" "=") symbol-end)))) (haskell-tng:font:multiline type - (rx line-start "type" word-end) - (rx line-start "type" word-end (group (+ anything))) - haskell-tng:indent-close) + (rx line-start "type" word-end) + (rx line-start "type" word-end (group (+ anything))) + haskell-tng:indent-close) ;; DeriveAnyClass ;; DerivingStrategies @@ -326,24 +326,24 @@ succeeds and may further restrict the FIND search limit." ;; EXT:DerivingVia ;; EXT:StandaloneDeriving (haskell-tng:font:multiline deriving - (rx word-start "deriving" word-end) - (rx word-start "deriving" word-end - (+ space) (group (opt (| "anyclass" "stock" "newtype") word-end)) - (* space) ?\( (group (* anything)) ?\)) - haskell-tng:indent-close) + (rx word-start "deriving" word-end) + (rx word-start "deriving" word-end + (+ space) (group (opt (| "anyclass" "stock" "newtype") word-end)) + (* space) ?\( (group (* anything)) ?\)) + haskell-tng:indent-close) (haskell-tng:font:multiline import - (rx line-start "import" word-end) - (rx line-start "import" word-end - (+ (not (any ?\( ))) - (opt "(" (group (+ anything)))) - haskell-tng:indent-close) + (rx line-start "import" word-end) + (rx line-start "import" word-end + (+ (not (any ?\( ))) + (opt "(" (group (+ anything)))) + haskell-tng:indent-close) (haskell-tng:font:multiline module - (rx line-start "module" word-end) - (rx line-start "module" word-end (group (+ anything)) - word-start "where" word-end) - haskell-tng:indent-close) + (rx line-start "module" word-end) + (rx line-start "module" word-end (group (+ anything)) + word-start "where" word-end) + haskell-tng:indent-close) (provide 'haskell-tng-font-lock) ;;; haskell-tng-font-lock.el ends here diff --git a/haskell-tng-layout.el b/haskell-tng-layout.el index b92570b..9e31ac4 100644 --- a/haskell-tng-layout.el +++ b/haskell-tng-layout.el @@ -54,23 +54,23 @@ Designed to be called repeatedly, managing its own caching." (haskell-tng-layout:rebuild-cache-full)) (let ((pos (point))) - (catch 'done - (let (breaks - closes) - (dolist (block haskell-tng-layout:cache) - (let ((open (car block)) - (close (cadr block)) - (lines (cddr block))) - ;;(message "BLOCK = %S (%s, %s, %s)" block open close lines) - (when (and (<= open pos) (<= pos close)) - (when (= open pos) - (throw 'done '("{"))) - (when (= close pos) - (push "}" closes)) - (dolist (line lines) - (when (= line pos) - (push ";" breaks)))))) - (append (reverse closes) (reverse breaks)))))) + (catch 'done + (let (breaks + closes) + (dolist (block haskell-tng-layout:cache) + (let ((open (car block)) + (close (cadr block)) + (lines (cddr block))) + ;;(message "BLOCK = %S (%s, %s, %s)" block open close lines) + (when (and (<= open pos) (<= pos close)) + (when (= open pos) + (throw 'done '("{"))) + (when (= close pos) + (push "}" closes)) + (dolist (line lines) + (when (= line pos) + (push ";" breaks)))))) + (append (reverse closes) (reverse breaks)))))) (defun haskell-tng-layout:rebuild-cache-full () (let (case-fold-search diff --git a/haskell-tng-lexer.el b/haskell-tng-lexer.el new file mode 100644 index 0000000..28fd6ce --- /dev/null +++ b/haskell-tng-lexer.el @@ -0,0 +1,157 @@ +;;; haskell-tng-lexer.el --- Haskell Lexer -*- lexical-binding: t -*- + +;; Copyright (C) 2018-2019 Tseen She +;; License: GPL 3 or any later version + +;;; Commentary: +;; +;; SMIE compatible lexer, (sadly) stateful in order to support virtual tokens. +;; See `haskell-tng-layout.el' for more details. +;; +;;; Code: + +(require 'smie) + +(require 'haskell-tng-font-lock) +(require 'haskell-tng-layout) + +;; The list of virtual tokens that must be played back at point, or `t' to +;; indicate that virtual tokens have already been played back at point and +;; normal lexing may continue. +(defvar-local haskell-tng-lexer:state nil) + +;; A cons cell of the last known direction and point when forward or backward +;; lexing was called. Used to invalidate `haskell-tng-lexer:state' during +;; read-only navigation. +(defvar-local haskell-tng-lexer:last nil) + +;; syntax-tables supported by SMIE +(defconst haskell-tng-lexer:fast-syntax + (rx (| (syntax open-parenthesis) + (syntax close-parenthesis) + (syntax string-quote) + (syntax string-delimiter)))) + +(defun haskell-tng-lexer:state-invalidation (_beg _end _pre-length) + "For use in `after-change-functions' to invalidate the state of +the lexer." + (when haskell-tng-lexer:state + (setq haskell-tng-lexer:state nil))) + +;; Implementation of `smie-forward-token' for Haskell, i.e. +;; +;; - Called with no argument should return a token and move to its end. +;; - If no token is found, return nil or the empty string. +;; - It can return nil when bumping into a parenthesis, which lets SMIE +;; use syntax-tables to handle them in efficient C code. +;; +;; https://www.gnu.org/software/emacs/manual/html_mono/elisp.html#SMIE-Lexer +;; +;; Note that this implementation is stateful as it can play back multiple +;; virtual tokens at a single point. This lexer could be made stateless if SMIE +;; were to support a 4th return type: a list of any of the above. +;; +;; Any changes to this function must be reflected in +;; `haskell-tng-lexer:backward-token'. +(defun haskell-tng-lexer:forward-token () + (unwind-protect + (let (case-fold-search) + (haskell-tng-lexer:check-last 'forward) + + (if (consp haskell-tng-lexer:state) + ;; continue replaying virtual tokens + (haskell-tng-lexer:replay-virtual) + + (forward-comment (point-max)) + + ;; TODO: performance. Only request virtuals when they make sense... + ;; e.g. on newlines, or following a WLDO (assuming a comment-aware + ;; lookback is fast). + (setq haskell-tng-lexer:state + (unless haskell-tng-lexer:state + (haskell-tng-layout:virtuals-at-point))) + + (cond + ;; new virtual tokens + (haskell-tng-lexer:state + (haskell-tng-lexer:replay-virtual)) + + ((eobp) nil) + + ;; syntax tables (supported by `smie-indent-forward-token') + ((looking-at haskell-tng-lexer:fast-syntax) nil) + + ;; regexps + ((or + ;; known identifiers + (looking-at haskell-tng:regexp:reserved) + ;; symbols + (looking-at (rx (+ (| (syntax word) (syntax symbol)))))) + (haskell-tng-lexer:last-match)) + + ;; single char + (t + (forward-char) + (string (char-before)))))) + + ;; save the state + (haskell-tng-lexer:set-last 'forward))) + +;; Implementation of `smie-backward-token' for Haskell, matching +;; `haskell-tng-lexer:forward-token'. +(defun haskell-tng-lexer:backward-token () + (unwind-protect + (let (case-fold-search) + (haskell-tng-lexer:check-last 'backward) + + (if (consp haskell-tng-lexer:state) + (haskell-tng-lexer:replay-virtual 'reverse) + + (setq haskell-tng-lexer:state + (unless haskell-tng-lexer:state + (haskell-tng-layout:virtuals-at-point))) + + (if haskell-tng-lexer:state + (haskell-tng-lexer:replay-virtual 'reverse) + + (forward-comment (- (point))) + (cond + ((bobp) nil) + ((looking-back haskell-tng-lexer:fast-syntax (- (point) 1)) nil) + ((or + (looking-back haskell-tng:regexp:reserved (- (point) 8)) + (looking-back (rx (+ (| (syntax word) (syntax symbol)))) + (line-beginning-position) 't)) + (haskell-tng-lexer:last-match 'reverse)) + (t + (forward-char -1) + (string (char-after))))))) + + (haskell-tng-lexer:set-last 'backward))) + +(defun haskell-tng-lexer:set-last (direction) + (setq haskell-tng-lexer:last (cons direction (point)))) + +(defun haskell-tng-lexer:check-last (direction) + (when (and haskell-tng-lexer:state + (not (equal haskell-tng-lexer:last (cons direction (point))))) + (setq haskell-tng-lexer:state nil))) + +(defun haskell-tng-lexer:replay-virtual (&optional reverse) + ";; read a virtual token from state, set 't when all done" + (unwind-protect + (if reverse + (unwind-protect + (car (last haskell-tng-lexer:state)) + (setq haskell-tng-lexer:state + (butlast haskell-tng-lexer:state))) + (pop haskell-tng-lexer:state)) + (unless haskell-tng-lexer:state + (setq haskell-tng-lexer:state 't)))) + +(defun haskell-tng-lexer:last-match (&optional reverse) + (goto-char (if reverse (match-beginning 0) (match-end 0))) + (match-string-no-properties 0)) + +(provide 'haskell-tng-lexer) +;;; haskell-tng-lexer.el ends here diff --git a/haskell-tng-smie.el b/haskell-tng-smie.el index ed89f69..45a2089 100644 --- a/haskell-tng-smie.el +++ b/haskell-tng-smie.el @@ -5,9 +5,8 @@ ;;; Commentary: ;; -;; SMIE lexer, precedence table (providing s-expression navigation), and -;; indentation rules. The lexer is stateful in order to support virtual tokens, -;; and Layout aware, see `haskell-tng-layout.el' for more details. +;; SMIE precedence table, providing s-expression navigation, and indentation +;; rules. ;; ;; Note that we don't support every aspect of the Haskell language. e.g. if we ;; had access to all the operators in scope, and their fixity, we could create @@ -22,147 +21,9 @@ (require 'smie) (require 'haskell-tng-font-lock) -(require 'haskell-tng-layout) +(require 'haskell-tng-lexer) -;; The list of virtual tokens that must be played back at point, or `t' to -;; indicate that virtual tokens have already been played back at point and -;; normal lexing may continue. -(defvar-local haskell-tng-smie:state nil) - -;; A cons cell of the last known direction and point when forward or backward -;; lexing was called. Used to invalidate `haskell-tng-smie:state' during -;; read-only navigation. -(defvar-local haskell-tng-smie:last nil) - -;; syntax-tables supported by SMIE -(defconst haskell-tng-smie:fast-syntax - (rx (| (syntax open-parenthesis) - (syntax close-parenthesis) - (syntax string-quote) - (syntax string-delimiter)))) - -(defun haskell-tng-smie:state-invalidation (_beg _end _pre-length) - "For use in `after-change-functions' to invalidate the state of -the lexer." - (when haskell-tng-smie:state - (setq haskell-tng-smie:state nil))) - -;; Implementation of `smie-forward-token' for Haskell, i.e. -;; -;; - Called with no argument should return a token and move to its end. -;; - If no token is found, return nil or the empty string. -;; - It can return nil when bumping into a parenthesis, which lets SMIE -;; use syntax-tables to handle them in efficient C code. -;; -;; https://www.gnu.org/software/emacs/manual/html_mono/elisp.html#SMIE-Lexer -;; -;; Note that this implementation is stateful as it can play back multiple -;; virtual tokens at a single point. This lexer could be made stateless if SMIE -;; were to support a 4th return type: a list of any of the above. -;; -;; Any changes to this function must be reflected in -;; `haskell-tng-smie:backward-token'. -(defun haskell-tng-smie:forward-token () - (unwind-protect - (let (case-fold-search) - (haskell-tng-smie:check-last 'forward) - - (if (consp haskell-tng-smie:state) - ;; continue replaying virtual tokens - (haskell-tng-smie:replay-virtual) - - (forward-comment (point-max)) - - ;; TODO: performance. Only request virtuals when they make sense... - ;; e.g. on newlines, or following a WLDO (assuming a comment-aware - ;; lookback is fast). - (setq haskell-tng-smie:state - (unless haskell-tng-smie:state - (haskell-tng-layout:virtuals-at-point))) - - (cond - ;; new virtual tokens - (haskell-tng-smie:state - (haskell-tng-smie:replay-virtual)) - - ((eobp) nil) - - ;; syntax tables (supported by `smie-indent-forward-token') - ((looking-at haskell-tng-smie:fast-syntax) nil) - - ;; regexps - ((or - ;; known identifiers - (looking-at haskell-tng:regexp:reserved) - ;; symbols - (looking-at (rx (+ (| (syntax word) (syntax symbol)))))) - (haskell-tng-smie:last-match)) - - ;; single char - (t - (forward-char) - (string (char-before)))))) - - ;; save the state - (haskell-tng-smie:set-last 'forward))) - -;; Implementation of `smie-backward-token' for Haskell, matching -;; `haskell-tng-smie:forward-token'. -(defun haskell-tng-smie:backward-token () - (unwind-protect - (let (case-fold-search) - (haskell-tng-smie:check-last 'backward) - - (if (consp haskell-tng-smie:state) - (haskell-tng-smie:replay-virtual 'reverse) - - (setq haskell-tng-smie:state - (unless haskell-tng-smie:state - (haskell-tng-layout:virtuals-at-point))) - - (if haskell-tng-smie:state - (haskell-tng-smie:replay-virtual 'reverse) - - (forward-comment (- (point))) - (cond - ((bobp) nil) - ((looking-back haskell-tng-smie:fast-syntax (- (point) 1)) nil) - ((or - (looking-back haskell-tng:regexp:reserved (- (point) 8)) - (looking-back (rx (+ (| (syntax word) (syntax symbol)))) - (line-beginning-position) 't)) - (haskell-tng-smie:last-match 'reverse)) - (t - (forward-char -1) - (string (char-after))))))) - - (haskell-tng-smie:set-last 'backward))) - -(defun haskell-tng-smie:set-last (direction) - (setq haskell-tng-smie:last (cons direction (point)))) - -(defun haskell-tng-smie:check-last (direction) - (when (and haskell-tng-smie:state - (not (equal haskell-tng-smie:last (cons direction (point))))) - (setq haskell-tng-smie:state nil))) - -(defun haskell-tng-smie:replay-virtual (&optional reverse) - ";; read a virtual token from state, set 't when all done" - (unwind-protect - (if reverse - (unwind-protect - (car (last haskell-tng-smie:state)) - (setq haskell-tng-smie:state - (butlast haskell-tng-smie:state))) - (pop haskell-tng-smie:state)) - (unless haskell-tng-smie:state - (setq haskell-tng-smie:state 't)))) - -(defun haskell-tng-smie:last-match (&optional reverse) - (goto-char (if reverse (match-beginning 0) (match-end 0))) - (match-string-no-properties 0)) - -;; TODO a haskell grammar +;; FIXME a haskell grammar ;; https://www.gnu.org/software/emacs/manual/html_mono/elisp.html#SMIE-Grammar (defvar haskell-tng-smie:grammar (smie-prec2->grammar @@ -176,7 +37,7 @@ the lexer." (exp (exp "+" exp) (exp "*" exp) ("(" exps ")") - ("{" exps "}")) + ("{" insts "}")) (exps (exps "," exps) (exp))) '((assoc ";")) '((assoc ",")) @@ -193,13 +54,13 @@ the lexer." (add-to-list 'after-change-functions - #'haskell-tng-smie:state-invalidation) + #'haskell-tng-lexer:state-invalidation) (smie-setup haskell-tng-smie:grammar haskell-tng-smie:rules - :forward-token #'haskell-tng-smie:forward-token - :backward-token #'haskell-tng-smie:backward-token)) + :forward-token #'haskell-tng-lexer:forward-token + :backward-token #'haskell-tng-lexer:backward-token)) (provide 'haskell-tng-smie) ;;; haskell-tng-smie.el ends here diff --git a/test/haskell-tng-layout-test.el b/test/haskell-tng-layout-test.el index 47ef63d..38f601f 100644 --- a/test/haskell-tng-layout-test.el +++ b/test/haskell-tng-layout-test.el @@ -21,7 +21,7 @@ (setq exit t) (push (string (char-after)) tokens) (forward-char))) - (s-join "" (reverse tokens)))) + (s-join "" (reverse tokens)))) (defun have-expected-layout (file) (haskell-tng-testutils:assert-file-contents diff --git a/test/haskell-tng-smie-test.el b/test/haskell-tng-lexer-test.el similarity index 53% copy from test/haskell-tng-smie-test.el copy to test/haskell-tng-lexer-test.el index 4d5457a..5559888 100644 --- a/test/haskell-tng-smie-test.el +++ b/test/haskell-tng-lexer-test.el @@ -1,4 +1,4 @@ -;;; haskell-tng-smie-test.el --- Tests for navigation and indentation -*- lexical-binding: t -*- +;;; haskell-tng-lexer-test.el --- Tests for navigation and indentation -*- lexical-binding: t -*- ;; Copyright (C) 2018-2019 Tseen She ;; License: GPL 3 or any later version @@ -11,16 +11,91 @@ (require 'haskell-tng-testutils "test/haskell-tng-testutils.el") +(ert-deftest haskell-tng-lexer-file-tests () + (should (have-expected-forward-lex (testdata "src/medley.hs"))) + (should (have-expected-forward-lex (testdata "src/layout.hs"))) + + (should (have-expected-backward-lex (testdata "src/medley.hs"))) + (should (have-expected-backward-lex (testdata "src/layout.hs"))) + ) + +(ert-deftest haskell-tng-lexer-state-invalidation-tests () + (with-temp-buffer + (insert-file-contents (testdata "src/layout.hs")) + (haskell-tng-mode) + + ;; three parses at this position will produce a virtual token and a real + ;; token, then move the point for another token. + (goto-char 317) + (should (equal (haskell-tng-lexer-test:indent-forward-token) ";")) + (should (equal (haskell-tng-lexer-test:indent-forward-token) "stkToLst")) + (should (equal (haskell-tng-lexer-test:indent-forward-token) "_(")) + + ;; repeating the above, but with a user edit, should reset the state + (goto-char 317) + (should (equal (haskell-tng-lexer-test:indent-forward-token) ";")) + (save-excursion + (goto-char (point-max)) + (insert " ")) + (should (equal (haskell-tng-lexer-test:indent-forward-token) ";")) + (should (equal (haskell-tng-lexer-test:indent-forward-token) "stkToLst")) + (should (equal (haskell-tng-lexer-test:indent-forward-token) "_(")) + + ;; repeating again, but jumping the lexer, should reset the state + (goto-char 317) + (should (equal (haskell-tng-lexer-test:indent-forward-token) ";")) + (goto-char 327) + (should (equal (haskell-tng-lexer-test:indent-forward-token) "MkStack")) + (goto-char 317) + (should (equal (haskell-tng-lexer-test:indent-forward-token) ";")) + (should (equal (haskell-tng-lexer-test:indent-forward-token) "stkToLst")) + (should (equal (haskell-tng-lexer-test:indent-forward-token) "_(")) + + ;; repeating those tests, but for the backward lexer + (goto-char 317) + (should (equal (haskell-tng-lexer-test:indent-backward-token) ";")) + (should (equal (haskell-tng-lexer-test:indent-backward-token) "_]")) + (should (equal (haskell-tng-lexer-test:indent-backward-token) "_[")) + + (goto-char 317) + (should (equal (haskell-tng-lexer-test:indent-backward-token) ";")) + (save-excursion + (goto-char (point-max)) + (insert " ")) + (should (equal (haskell-tng-lexer-test:indent-backward-token) ";")) + (should (equal (haskell-tng-lexer-test:indent-backward-token) "_]")) + (should (equal (haskell-tng-lexer-test:indent-backward-token) "_[")) + + (goto-char 317) + (should (equal (haskell-tng-lexer-test:indent-backward-token) ";")) + (goto-char 327) + (should (equal (haskell-tng-lexer-test:indent-backward-token) "_(")) + (goto-char 317) + (should (equal (haskell-tng-lexer-test:indent-backward-token) ";")) + (should (equal (haskell-tng-lexer-test:indent-backward-token) "_]")) + (should (equal (haskell-tng-lexer-test:indent-backward-token) "_[")) + + ;; jumping between forward and backward at point should reset state + (goto-char 317) + (should (equal (haskell-tng-lexer-test:indent-forward-token) ";")) + (should (equal (haskell-tng-lexer-test:indent-backward-token) ";")) + (should (equal (haskell-tng-lexer-test:indent-forward-token) ";")) + (should (equal (haskell-tng-lexer-test:indent-backward-token) ";")) + )) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; SMIE testing utilities + ;; copy/pasta of `smie-indent-forward-token' but rendering lexed tokens in a way ;; more ammenable to regression testing (e.g. syntax table usage) -(defun haskell-tng-smie-test:indent-forward-token () +(defun haskell-tng-lexer-test:indent-forward-token () (let ((tok (funcall smie-forward-token-function))) (cond ((< 0 (length tok)) tok) ((eobp) nil) ((looking-at (rx (| (syntax open-parenthesis) (syntax close-parenthesis)))) - (concat "_" (haskell-tng-smie:last-match))) + (concat "_" (haskell-tng-lexer:last-match))) ((looking-at (rx (| (syntax string-quote) (syntax string-delimiter)))) (let ((start (point))) @@ -29,7 +104,7 @@ (t (error "Bumped into unknown token"))))) ;; same as above, but for `smie-indent-backward-token' -(defun haskell-tng-smie-test:indent-backward-token () +(defun haskell-tng-lexer-test:indent-backward-token () (let ((tok (funcall smie-backward-token-function))) (cond ((< 0 (length tok)) tok) @@ -37,7 +112,7 @@ ((looking-back (rx (| (syntax open-parenthesis) (syntax close-parenthesis))) (- (point) 1)) - (concat "_" (haskell-tng-smie:last-match 'reverse))) + (concat "_" (haskell-tng-lexer:last-match 'reverse))) ((looking-back (rx (| (syntax string-quote) (syntax string-delimiter))) (- (point) 1)) @@ -46,19 +121,19 @@ (concat "_" (buffer-substring-no-properties (point) start)))) (t (error "Bumped into unknown token"))))) -(defun haskell-tng-smie-test:tokens (&optional reverse) +(defun haskell-tng-lexer-test:tokens (&optional reverse) "Lex the current buffer using SMIE and return the list of lines, where each line is a list of tokens. When called interactively, shows the tokens in a buffer." - (let ((lines (list nil)) - quit) + (let (lines quit) + (push nil lines) (goto-char (if reverse (point-max) (point-min))) (while (not quit) (let* ((start (point)) (token (if reverse - (haskell-tng-smie-test:indent-backward-token) - (haskell-tng-smie-test:indent-forward-token)))) + (haskell-tng-lexer-test:indent-backward-token) + (haskell-tng-lexer-test:indent-forward-token)))) (let ((line-diff (- (line-number-at-pos (point)) (line-number-at-pos start)))) (unless (= line-diff 0) @@ -71,98 +146,23 @@ When called interactively, shows the tokens in a buffer." lines (reverse (--map (reverse it) lines))))) -(defun haskell-tng-smie-test:tokens-to-string (lines) +(defun haskell-tng-lexer-test:tokens-to-string (lines) (concat (s-join "\n" (--map (s-join " " it) lines)) "\n")) (defun have-expected-forward-lex (file) (haskell-tng-testutils:assert-file-contents file #'haskell-tng-mode - (lambda () (haskell-tng-smie-test:tokens-to-string - (haskell-tng-smie-test:tokens))) + (lambda () (haskell-tng-lexer-test:tokens-to-string + (haskell-tng-lexer-test:tokens))) "lexer")) (defun have-expected-backward-lex (file) (haskell-tng-testutils:assert-file-contents file #'haskell-tng-mode - (lambda () (haskell-tng-smie-test:tokens-to-string - (haskell-tng-smie-test:tokens 'reverse))) + (lambda () (haskell-tng-lexer-test:tokens-to-string + (haskell-tng-lexer-test:tokens 'reverse))) "lexer")) -(ert-deftest haskell-tng-smie-file-tests () - ;;(should (have-expected-forward-lex (testdata "src/medley.hs"))) - ;;(should (have-expected-forward-lex (testdata "src/layout.hs"))) - - (should (have-expected-backward-lex (testdata "src/medley.hs"))) - (should (have-expected-backward-lex (testdata "src/layout.hs"))) - ) - -(ert-deftest haskell-tng-smie-state-invalidation-tests () - (with-temp-buffer - (insert-file-contents (testdata "src/layout.hs")) - (haskell-tng-mode) - - ;; three parses at this position will produce a virtual token and a real - ;; token, then move the point for another token. - (goto-char 317) - (should (equal (haskell-tng-smie-test:indent-forward-token) ";")) - (should (equal (haskell-tng-smie-test:indent-forward-token) "stkToLst")) - (should (equal (haskell-tng-smie-test:indent-forward-token) "_(")) - - ;; repeating the above, but with a user edit, should reset the state - (goto-char 317) - (should (equal (haskell-tng-smie-test:indent-forward-token) ";")) - (save-excursion - (goto-char (point-max)) - (insert " ")) - (should (equal (haskell-tng-smie-test:indent-forward-token) ";")) - (should (equal (haskell-tng-smie-test:indent-forward-token) "stkToLst")) - (should (equal (haskell-tng-smie-test:indent-forward-token) "_(")) - - ;; repeating again, but jumping the lexer, should reset the state - (goto-char 317) - (should (equal (haskell-tng-smie-test:indent-forward-token) ";")) - (goto-char 327) - (should (equal (haskell-tng-smie-test:indent-forward-token) "MkStack")) - (goto-char 317) - (should (equal (haskell-tng-smie-test:indent-forward-token) ";")) - (should (equal (haskell-tng-smie-test:indent-forward-token) "stkToLst")) - (should (equal (haskell-tng-smie-test:indent-forward-token) "_(")) - - ;; repeating those tests, but for the backward lexer - (goto-char 317) - (should (equal (haskell-tng-smie-test:indent-backward-token) ";")) - (should (equal (haskell-tng-smie-test:indent-backward-token) "_]")) - (should (equal (haskell-tng-smie-test:indent-backward-token) "_[")) - - (goto-char 317) - (should (equal (haskell-tng-smie-test:indent-backward-token) ";")) - (save-excursion - (goto-char (point-max)) - (insert " ")) - (should (equal (haskell-tng-smie-test:indent-backward-token) ";")) - (should (equal (haskell-tng-smie-test:indent-backward-token) "_]")) - (should (equal (haskell-tng-smie-test:indent-backward-token) "_[")) - - (goto-char 317) - (should (equal (haskell-tng-smie-test:indent-backward-token) ";")) - (goto-char 327) - (should (equal (haskell-tng-smie-test:indent-backward-token) "_(")) - (goto-char 317) - (should (equal (haskell-tng-smie-test:indent-backward-token) ";")) - (should (equal (haskell-tng-smie-test:indent-backward-token) "_]")) - (should (equal (haskell-tng-smie-test:indent-backward-token) "_[")) - - ;; jumping between forward and backward at point should reset state - (goto-char 317) - (should (equal (haskell-tng-smie-test:indent-forward-token) ";")) - (should (equal (haskell-tng-smie-test:indent-backward-token) ";")) - (should (equal (haskell-tng-smie-test:indent-forward-token) ";")) - (should (equal (haskell-tng-smie-test:indent-backward-token) ";")) - )) - -;; ideas for an indentation tester -;; https://github.com/elixir-editors/emacs-elixir/blob/master/test/test-helper.el#L52-L63 - -;;; haskell-tng-smie-test.el ends here +;;; haskell-tng-lexer-test.el ends here diff --git a/test/haskell-tng-smie-test.el b/test/haskell-tng-smie-test.el index 4d5457a..51f4752 100644 --- a/test/haskell-tng-smie-test.el +++ b/test/haskell-tng-smie-test.el @@ -1,4 +1,4 @@ -;;; haskell-tng-smie-test.el --- Tests for navigation and indentation -*- lexical-binding: t -*- +;;; haskell-tng-lexer-test.el --- Tests for navigation and indentation -*- lexical-binding: t -*- ;; Copyright (C) 2018-2019 Tseen She ;; License: GPL 3 or any later version @@ -11,158 +11,13 @@ (require 'haskell-tng-testutils "test/haskell-tng-testutils.el") -;; copy/pasta of `smie-indent-forward-token' but rendering lexed tokens in a way -;; more ammenable to regression testing (e.g. syntax table usage) -(defun haskell-tng-smie-test:indent-forward-token () - (let ((tok (funcall smie-forward-token-function))) - (cond - ((< 0 (length tok)) tok) - ((eobp) nil) - ((looking-at (rx (| (syntax open-parenthesis) - (syntax close-parenthesis)))) - (concat "_" (haskell-tng-smie:last-match))) - ((looking-at (rx (| (syntax string-quote) - (syntax string-delimiter)))) - (let ((start (point))) - (forward-sexp 1) - (concat "_" (buffer-substring-no-properties start (point))))) - (t (error "Bumped into unknown token"))))) - -;; same as above, but for `smie-indent-backward-token' -(defun haskell-tng-smie-test:indent-backward-token () - (let ((tok (funcall smie-backward-token-function))) - (cond - ((< 0 (length tok)) tok) - ((bobp) nil) - ((looking-back (rx (| (syntax open-parenthesis) - (syntax close-parenthesis))) - (- (point) 1)) - (concat "_" (haskell-tng-smie:last-match 'reverse))) - ((looking-back (rx (| (syntax string-quote) - (syntax string-delimiter))) - (- (point) 1)) - (let ((start (point))) - (backward-sexp 1) - (concat "_" (buffer-substring-no-properties (point) start)))) - (t (error "Bumped into unknown token"))))) - -(defun haskell-tng-smie-test:tokens (&optional reverse) - "Lex the current buffer using SMIE and return the list of lines, -where each line is a list of tokens. - -When called interactively, shows the tokens in a buffer." - (let ((lines (list nil)) - quit) - (goto-char (if reverse (point-max) (point-min))) - (while (not quit) - (let* ((start (point)) - (token (if reverse - (haskell-tng-smie-test:indent-backward-token) - (haskell-tng-smie-test:indent-forward-token)))) - (let ((line-diff (- (line-number-at-pos (point)) - (line-number-at-pos start)))) - (unless (= line-diff 0) - (setq lines (append (-repeat (abs line-diff) nil) lines)))) - (if (and (not token) (if reverse (bobp) (eobp))) - (setq quit 't) - (unless (s-blank? token) - (push token (car lines)))))) - (if reverse - lines - (reverse (--map (reverse it) lines))))) - -(defun haskell-tng-smie-test:tokens-to-string (lines) - (concat (s-join "\n" (--map (s-join " " it) lines)) "\n")) - -(defun have-expected-forward-lex (file) - (haskell-tng-testutils:assert-file-contents - file - #'haskell-tng-mode - (lambda () (haskell-tng-smie-test:tokens-to-string - (haskell-tng-smie-test:tokens))) - "lexer")) - -(defun have-expected-backward-lex (file) - (haskell-tng-testutils:assert-file-contents - file - #'haskell-tng-mode - (lambda () (haskell-tng-smie-test:tokens-to-string - (haskell-tng-smie-test:tokens 'reverse))) - "lexer")) - (ert-deftest haskell-tng-smie-file-tests () - ;;(should (have-expected-forward-lex (testdata "src/medley.hs"))) - ;;(should (have-expected-forward-lex (testdata "src/layout.hs"))) - - (should (have-expected-backward-lex (testdata "src/medley.hs"))) - (should (have-expected-backward-lex (testdata "src/layout.hs"))) + ;; FIXME tests for s-expressions + ;; (should (have-expected-forward-lex (testdata "src/medley.hs"))) + ;; (should (have-expected-forward-lex (testdata "src/layout.hs"))) ) -(ert-deftest haskell-tng-smie-state-invalidation-tests () - (with-temp-buffer - (insert-file-contents (testdata "src/layout.hs")) - (haskell-tng-mode) - - ;; three parses at this position will produce a virtual token and a real - ;; token, then move the point for another token. - (goto-char 317) - (should (equal (haskell-tng-smie-test:indent-forward-token) ";")) - (should (equal (haskell-tng-smie-test:indent-forward-token) "stkToLst")) - (should (equal (haskell-tng-smie-test:indent-forward-token) "_(")) - - ;; repeating the above, but with a user edit, should reset the state - (goto-char 317) - (should (equal (haskell-tng-smie-test:indent-forward-token) ";")) - (save-excursion - (goto-char (point-max)) - (insert " ")) - (should (equal (haskell-tng-smie-test:indent-forward-token) ";")) - (should (equal (haskell-tng-smie-test:indent-forward-token) "stkToLst")) - (should (equal (haskell-tng-smie-test:indent-forward-token) "_(")) - - ;; repeating again, but jumping the lexer, should reset the state - (goto-char 317) - (should (equal (haskell-tng-smie-test:indent-forward-token) ";")) - (goto-char 327) - (should (equal (haskell-tng-smie-test:indent-forward-token) "MkStack")) - (goto-char 317) - (should (equal (haskell-tng-smie-test:indent-forward-token) ";")) - (should (equal (haskell-tng-smie-test:indent-forward-token) "stkToLst")) - (should (equal (haskell-tng-smie-test:indent-forward-token) "_(")) - - ;; repeating those tests, but for the backward lexer - (goto-char 317) - (should (equal (haskell-tng-smie-test:indent-backward-token) ";")) - (should (equal (haskell-tng-smie-test:indent-backward-token) "_]")) - (should (equal (haskell-tng-smie-test:indent-backward-token) "_[")) - - (goto-char 317) - (should (equal (haskell-tng-smie-test:indent-backward-token) ";")) - (save-excursion - (goto-char (point-max)) - (insert " ")) - (should (equal (haskell-tng-smie-test:indent-backward-token) ";")) - (should (equal (haskell-tng-smie-test:indent-backward-token) "_]")) - (should (equal (haskell-tng-smie-test:indent-backward-token) "_[")) - - (goto-char 317) - (should (equal (haskell-tng-smie-test:indent-backward-token) ";")) - (goto-char 327) - (should (equal (haskell-tng-smie-test:indent-backward-token) "_(")) - (goto-char 317) - (should (equal (haskell-tng-smie-test:indent-backward-token) ";")) - (should (equal (haskell-tng-smie-test:indent-backward-token) "_]")) - (should (equal (haskell-tng-smie-test:indent-backward-token) "_[")) - - ;; jumping between forward and backward at point should reset state - (goto-char 317) - (should (equal (haskell-tng-smie-test:indent-forward-token) ";")) - (should (equal (haskell-tng-smie-test:indent-backward-token) ";")) - (should (equal (haskell-tng-smie-test:indent-forward-token) ";")) - (should (equal (haskell-tng-smie-test:indent-backward-token) ";")) - )) - ;; ideas for an indentation tester ;; https://github.com/elixir-editors/emacs-elixir/blob/master/test/test-helper.el#L52-L63 -;;; haskell-tng-smie-test.el ends here +;;; haskell-tng-lexer-test.el ends here diff --git a/test/haskell-tng-testutils.el b/test/haskell-tng-testutils.el index 86c9f0a..a4491b8 100644 --- a/test/haskell-tng-testutils.el +++ b/test/haskell-tng-testutils.el @@ -16,7 +16,7 @@ default-directory))) (defun haskell-tng-testutils:assert-file-contents - (file mode to-string suffix) + (file mode to-string suffix) "For FILE, enable MODE and run TO-STRING and compare with the golden data in FILE.SUFFIX. Will fail and write out the expected version to FILE.SUFFIX." @@ -25,9 +25,9 @@ Will fail and write out the expected version to FILE.SUFFIX." (insert-file-contents golden) (buffer-string))) (got (with-temp-buffer - (insert-file-contents file) - (funcall mode) - (funcall to-string)))) + (insert-file-contents file) + (funcall mode) + (funcall to-string)))) (or (equal got expected) ;; writes out the new version on failure (progn