I've made a new lexer, since I had a hard time making a parser. Here it it:
import std/strutils
import std/os
let file = readFile(paramStr(1))
type
tokenName = enum
default, unknown, ident, str, integer, double, keyword, operator,
seperator
token = object
label: tokenName
value: string
abstNode = object
head: token
body: seq[abst]
abst = ref abstNode
let equivParams = [
(name: "none", equiv_fn: proc (x: token): bool =
return [default, unknown].contains(x.label)
),
(name: "identifier", equiv_fn: proc (x: token): bool =
return x.label == ident
),
(name: "type", equiv_fn: proc (x: token): bool =
return ["int", "char", "string"].contains(x.value)
),
(name: "=", equiv_fn: proc (x: token): bool =
return x.label == operator and x.value == "="
),
(name: ":", equiv_fn: proc (x: token): bool =
return x.label == seperator and x.value == ":"
)
]
var equivParamValues: seq[string]
for v in equivParams:
equivParamValues.add(v.name)
proc tkn(label: tokenName, value: string): token =
var t: token
t.label = label
t.value = value
return t
proc newAst(ast_tk: token): abst =
var nw: abst
nw.head = ast_tk
return nw
var currentTk: token
var tokens: seq[token]
const
seperators = ['(','[', '{', '}', ']', ')', ':', ';',]
head_seperators = ['(','[', '{']
end_seperators = [')', ']', '}']
operators = ["=", "+", "-", "/", "*", "^", "==", "++", "--", "<", ">"]
operatorStart = ['=','+','-','/', '*', '^', '<', '>']
var keywordStrings = ["let","var", "const", "fn", "return"]
proc reset(tk: var token) =
tk.label = default
tk.value = ""
proc unknownType(ch: char) =
case ch:
of '\"':
currentTk.label = str
of Digits:
currentTk.label = integer
currentTk.value = $ch
tokens.add(currentTk)
of IdentStartChars:
currentTk.label = ident
currentTk.value = $ch
of Whitespace:
return
of seperators:
currentTk.label = seperator
currentTk.value = $ch
tokens.add(currentTk)
currentTk.reset()
of operatorStart:
currentTk.label = operator
currentTk.value = $ch
else:
return
proc identType(ch: char) =
case ch
of operatorStart:
tokens.add(currentTk)
currentTk.reset()
currentTk.label = operator
currentTk.value = $ch
return
of seperators:
tokens.add(currentTk)
currentTk.reset()
currentTk.label = seperator
currentTk.value = $ch
tokens.add(currentTk)
currentTk.reset()
return
of Whitespace:
if keywordStrings.contains(currentTk.value):
currentTk.label = keyword
tokens.add(currentTk)
currentTk.reset()
return
of IdentChars:
currentTk.value.add(ch)
return
else:
return
proc strType(ch: char) =
if ch == '\"':
tokens.add(currentTk)
currentTk.reset()
else:
currentTk.value.add(ch)
proc numType(ch: char) =
case ch
of Digits:
currentTk.value.add(ch)
of '.':
currentTk.value.add(ch)
currentTk.label = double
of Whitespace:
tokens.add(currentTk)
currentTk.reset()
of operatorStart:
tokens.delete(tokens.len() - 1)
tokens.add(currentTk)
currentTk.reset()
currentTk.label = operator
currentTk.value = $ch
return
of seperators:
tokens.delete(tokens.len() - 1)
tokens.add(currentTk)
currentTk.reset()
currentTk.label = seperator
currentTk.value = $ch
tokens.add(currentTk)
currentTk.reset()
of IdentStartChars:
tokens.add(currentTk)
currentTk.label = ident
currentTk.value = $ch
else:
discard
proc opType(ch: char) =
if operators.contains($ch):
tokens.add(currentTk)
currentTk.reset()
if Whitespace.contains(ch):
if operators.contains(currentTk.value):
tokens.add(currentTk)
currentTk.reset()
else:
currentTk.value.add(ch)
for ch in file:
if currentTk.label == default:
unknownType(ch)
else:
case currentTk.label
of default:
break
of str:
strType(ch)
of ident:
identType(ch)
of integer, double:
numType(ch)
of operator:
opType(ch)
else:
continue
proc echoTks(tkns: seq[token]) =
for t in tokens:
echo t
var idx: int = 0
proc parseTkns(tkns: var seq[token], params: seq[string]): abst =
var ast: abst
if params.len() == 0:
while(true):
if tkns[idx].value in ["let", "var", "const"]:
ast.body.add(parseTkns(tkns, @["identifier", ":", "type"]))
idx += 1
if idx == tkns.len():
break
else:
var par: seq[string] = params
while(par.len() > idx):
echo par[idx]
if
equivParams[equivParamValues.find(par[idx])].equiv_fn(tkns[idx]):
ast.body.add(newAst(tkns[idx]))
idx += 1
return ast
var ast = parseTkns(tokens, @[])
Run
, And when I run it, it tells me that:
Traceback (most recent call last) srcmain.nim(213) main srcmain.nim(200)
parseTkns SIGSEGV: Illegal storage access. (Attempt to read from nil?).
Does anyone know how to fix this, or another way to do this? I'm sure it has to
do with the recursion on the lines.
