branch: externals/parser-generator
commit c299371a7453b208a123851cfa75047ede68bbc8
Author: Christian Johansson <[email protected]>
Commit: Christian Johansson <[email protected]>
Started work on refactoring lex-analyzer into a state-based lex-analyzer
---
TODO.md | 2 +
parser-generator-lex-analyzer.el | 107 +++++++++++++++++++----------
test/parser-generator-lex-analyzer-test.el | 19 +++--
3 files changed, 85 insertions(+), 43 deletions(-)
diff --git a/TODO.md b/TODO.md
index 952c7d226e..12403ad50f 100644
--- a/TODO.md
+++ b/TODO.md
@@ -8,6 +8,8 @@ Functions (with validations) to set global variables:
* parser-generator--global-attributes
* parser-generator--global-declaration
+State-based lex-analyzer
+
## LR-Parser
Functions (with validations) to set global variables:
diff --git a/parser-generator-lex-analyzer.el b/parser-generator-lex-analyzer.el
index 5422cc7e36..9f1ff4f4c7 100644
--- a/parser-generator-lex-analyzer.el
+++ b/parser-generator-lex-analyzer.el
@@ -31,14 +31,24 @@
"Index in lex-analyzer.")
(defvar
- parser-generator-lex-analyzer--reset-function
- nil
- "Function used when resetting lex-analyzer.")
+ parser-generation-lex-analyzer--index-init
+ 1
+ "Initial value of index.")
(defvar-local
- parser-generator-lex-analyzer--move-to-index-flag
+ parser-generation-lex-analyzer--state
+ nil
+ "State of lex-analyzer.")
+
+(defvar
+ parser-generator-lex-analyzer--state-init
nil
- "Non-nil means move index to value.")
+ "Initial value of state.")
+
+(defvar
+ parser-generator-lex-analyzer--reset-function
+ nil
+ "Function used when resetting lex-analyzer.")
;; Functions
@@ -55,10 +65,11 @@
(let ((meta-information))
(condition-case error
(progn
- (setq meta-information
- (funcall
- parser-generator-lex-analyzer--get-function
- token)))
+ (setq
+ meta-information
+ (funcall
+ parser-generator-lex-analyzer--get-function
+ token)))
(error
(signal
'error
@@ -88,6 +99,7 @@
(let ((look-ahead)
(look-ahead-length 0)
(index parser-generator-lex-analyzer--index)
+ (state parser-generation-lex-analyzer--state)
(k (max
1
parser-generator--look-ahead-number)))
@@ -96,22 +108,30 @@
k)
(condition-case error
(progn
- (setq-local
- parser-generator-lex-analyzer--move-to-index-flag
- nil)
- (let ((next-look-ahead
+ (let* ((result-list
(funcall
parser-generator-lex-analyzer--function
- index)))
- (if parser-generator-lex-analyzer--move-to-index-flag
- (setq
- index
- parser-generator-lex-analyzer--move-to-index-flag)
- (if next-look-ahead
+ index
+ state))
+ (token
+ (nth 0 result-list))
+ (move-to-index-flag
+ (nth 1 result-list))
+ (new-state
+ (nth 3 result-list)))
+ (if move-to-index-flag
+ (progn
+ (setq
+ index
+ move-to-index-flag)
+ (setq
+ state
+ new-state))
+ (if token
(progn
- (unless (listp (car next-look-ahead))
- (setq next-look-ahead (list next-look-ahead)))
- (dolist (next-look-ahead-item next-look-ahead)
+ (unless (listp (car token))
+ (setq token (list token)))
+ (dolist (next-look-ahead-item token)
(when (<
look-ahead-length
k)
@@ -141,25 +161,34 @@
(while continue
(condition-case error
(progn
- (setq-local
- parser-generator-lex-analyzer--move-to-index-flag
- nil)
- (let ((token
- (funcall
- parser-generator-lex-analyzer--function
- parser-generator-lex-analyzer--index)))
- (if parser-generator-lex-analyzer--move-to-index-flag
+ (let* ((result-list
+ (funcall
+ parser-generator-lex-analyzer--function
+ parser-generator-lex-analyzer--index
+ parser-generator-lex-analyzer--state))
+ (token
+ (nth 0 result-list))
+ (move-to-index-flag
+ (nth 1 result-list))
+ (new-index
+ (nth 2 result-list))
+ (new-state
+ (nth 3 result-list)))
+ (if move-to-index-flag
(progn
(setq-local
parser-generator-lex-analyzer--index
- parser-generator-lex-analyzer--move-to-index-flag))
+ move-to-index-flag)
+ (setq-local
+ parser-generator-lex-analyzer--state
+ new-state))
+ (setq
+ parser-generator-lex-analyzer--index
+ new-index)
(when token
(unless (listp (car token))
(setq token (list token)))
(let ((first-token (car token)))
- (setq
- parser-generator-lex-analyzer--index
- (cdr (cdr first-token)))
(push
first-token
tokens)))
@@ -168,14 +197,20 @@
nil))))
(error
(error
- "Lex-analyze failed to pop token at %s, error: %s"
+ "Lex-analyze failed to pop token at %s %s, error: %s"
parser-generator-lex-analyzer--index
+ parser-generator-lex-analyzer--state
(car (cdr error))))))
(nreverse tokens)))
(defun parser-generator-lex-analyzer--reset ()
"Reset lex-analyzer."
- (setq parser-generator-lex-analyzer--index 1)
+ (setq
+ parser-generator-lex-analyzer--index
+ parser-generation-lex-analyzer--index-init)
+ (setq
+ parser-generator-lex-analyzer--state
+ parser-generator-lex-analyzer--state-init)
(when parser-generator-lex-analyzer--reset-function
(funcall parser-generator-lex-analyzer--reset-function)))
diff --git a/test/parser-generator-lex-analyzer-test.el
b/test/parser-generator-lex-analyzer-test.el
index 2bbeb0dad1..a0c85eaff4 100644
--- a/test/parser-generator-lex-analyzer-test.el
+++ b/test/parser-generator-lex-analyzer-test.el
@@ -24,7 +24,7 @@
(parser-generator-lex-analyzer--peek-next-look-ahead))
(setq
parser-generator-lex-analyzer--function
- (lambda (index)
+ (lambda (index _state)
(let* ((string '(("a" 1 . 2) ("b" 2 . 3) ("c" 3 . 4) ("d" 4 . 5)))
(string-length (length string))
(max-index index)
@@ -36,7 +36,7 @@
(setq next-token (nth (1- index) string))
(push next-token tokens)
(setq index (1+ index)))
- (nreverse tokens))))
+ (list (nreverse tokens) nil nil nil))))
(should-error
(parser-generator-lex-analyzer--peek-next-look-ahead))
(parser-generator-lex-analyzer--reset)
@@ -46,6 +46,7 @@
(message "Passed failing lex analysis")
(setq parser-generator--look-ahead-number 1)
+ (parser-generator-lex-analyzer--peek-next-look-ahead)
(should
(equal
'(("a" 1 . 2))
@@ -65,7 +66,7 @@
(setq
parser-generator-lex-analyzer--function
- (lambda (index)
+ (lambda (index _state)
(let* ((string '(("a" 1 . 2) ("b" 2 . 3) ("c" 3 . 4) ("d" 4 . 5)))
(string-length (length string))
(max-index index)
@@ -79,7 +80,7 @@
(error "Invalid token: %s" next-token))
(push next-token tokens)
(setq index (1+ index)))
- (nreverse tokens))))
+ (list (nreverse tokens) nil nil nil))))
(should-error
(parser-generator-lex-analyzer--peek-next-look-ahead))
@@ -98,17 +99,19 @@
(parser-generator-lex-analyzer--pop-token))
(setq
parser-generator-lex-analyzer--function
- (lambda (index)
+ (lambda (index _state)
(let* ((string '(("a" 1 . 2) ("b" 2 . 3)))
(string-length (length string))
(max-index index)
- (tokens))
+ (tokens)
+ (new-index))
(while (and
(< (1- index) string-length)
(< (1- index) max-index))
(push (nth (1- index) string) tokens)
+ (setq new-index (cdr (cdr (nth (1- index) string))))
(setq index (1+ index)))
- (nreverse tokens))))
+ (list (nreverse tokens) nil new-index nil))))
(should-error
(parser-generator-lex-analyzer--pop-token))
(parser-generator-lex-analyzer--reset)
@@ -122,10 +125,12 @@
(equal
'(("a" 1 . 2))
(parser-generator-lex-analyzer--pop-token)))
+ (message "was there")
(should
(equal
'(("b" 2 . 3))
(parser-generator-lex-analyzer--pop-token)))
+ (message "was here")
(should
(equal
nil
