I realized that what I foresee the way to implement may not be what you get.
Since I have not coded nim for some time, I decided to do an exercise... but it
took longer than I thought T^T...
Anyway, the following program have done
1. string option with choices
2. int option range validation
3. multi-string option with choices (not demanded)
4. composition of options (not exactly inheritance)
5. runtime parsing
6. default value for form "\--option"
7. need a bit of skill to read error message (talk more below)
First, have a taste of parser combinators. It seems a lot code, but the hard
part is that honeycomb is missing the combinators I want.
import honeycomb
import std/json
import std/sets
import std/sugar
import std/sequtils
import std/strutils
import std/strformat
# -------------------------------------------------------------
# general commbinators
proc succeedWith[T](x: T): Parser[T] =
# common combinator missing in honeycomb?
# nop[void]().result(x) ?
createParser(T): succeed(input, x, input)
proc failWith[T](msg: openArray[string]): Parser[T] =
# common combinator missing in honeycomb?
let expected = @msg
createParser(T): fail(input, expected, input)
proc choice[T](ps: openArray[Parser[T]], desc: string = "no options"):
Parser[T] =
# wrap oneOf(varargs) into choices(openArray)
if len(ps) == 0: return failWith[T]([desc])
result = ps[1]
for i in 1 .. ps.high: result = result | ps[i]
proc branch[T1, T2, T](
ps: openArray[(Parser[T1], Parser[T2])],
f: proc(t1: T1, t2: T2): T): Parser[T] =
runnableExamples:
# use like if(...) ... elseif(...) ... elseif(...) ...,
# if the first parser match, go to the second parser and do not
backtrack other branches.
let oct = c('0'..'7')
let bin = c('0'..'1')
let p = branch([
(s("0b"), bin.atLeast(1)),
(s("0"), oct.atLeast(1)),
(s(""), digit.atLeast(1)),
], proc(base: string, ds: seq[char]): int =
let b = case base:
of "0b": 2
of "0": 8
else: 10
for d in ds: result = result * b + ord(d) - ord('0')
)
let r = p.parse("0bFFFF")
assert r.kind == failure
let copy = @ps
createParser(T):
var expects: seq[string]
for (cond, body) in copy:
let r1 = cond.parse(input)
case r1.kind:
of failure:
expects.add r1.expected
of success:
let r2 = body.parse(r1.tail)
case r2.kind:
of failure: return fail(input, r2.expected, input)
of success: return succeed(input, f(r1.value, r2.value), r2.tail)
fail(input, expects, input)
proc sepBy[T, T2](p1: Parser[T], p2: Parser[T2]): Parser[seq[T]] =
## [<p1> (<p2> <p1>)*]
runnableExamples:
let p = wordParser().sepBy(c(','))
let r = p.parse("abc,xyz")
assert r.kind == success
assert r.value == @["abc", "xyz"]
assert r.tail == ""
runnableExamples:
# this will fail because of the trailing space
let p = wordParser().sepBy(c(' '))
let r = p.parse("abc xyz ")
assert r.kind == failure
createParser(seq[T]):
var res: seq[T]
var r1: ParseResult[T]
var r2: ParseResult[T2]
r1 = p1.parse(input)
case r1.kind:
of failure:
return succeed(input, res, input)
of success:
res.add r1.value
# alternatively parse p2, p1, p2, p1...
while true:
r2 = p2.parse(r1.tail)
case r2.kind:
of failure:
return succeed(input, res, r1.tail)
of success:
r1 = p1.parse(r2.tail)
case r1.kind:
of failure:
# a success of p2 follow by fail of p1 result in whole fail
# return fail(input, r1.expected, input)
var expects = @[fmt"parse failure near `{r2.tail}`"]
expects.add r1.expected
return fail(input, expects, input)
of success:
res.add r1.value
proc wordParser(): Parser[string] =
runnableExamples:
let r = wordParser().parse("abc xyz")
assert r.kind == success
assert r.value = "abc"
assert r.tail = " xyz"
alphanumeric.atLeast(1).map(cs => cs.join(""))
proc intParser(): Parser[int] =
# todo: this accept leading zeros e.g. 007, which is not a strictly
correct grammar of integer
digit.atLeast(1).map(cs => parseInt(cs.join("")))
# -------------------------------------------------------------
# application specific patterns
type OptionParser = Parser[JsonNode]
let dash = s("--")
let wsp = regex(r"\s*")
let sep = c(':') | c('=') # for fun
proc strOpt(name: string, opts: openArray[string]): OptionParser =
## match --<name>:<option>
let copy = opts.toHashSet
let option = wordParser().validate(w => w in copy,
fmt"invalid option for {name}")
let full = dash >> s(name) >> sep >> option
full.map(s => %*{name: s})
proc strOpt(name: string, default: string, opts: openArray[string]):
OptionParser =
## match --<name>
## match --<name>:<option>
let copy = opts.toHashSet
let option = wordParser().validate(w => w in copy, fmt"invalid option for
{name}")
let full = dash >> s(name) >> branch([
(sep, option),
# last case always success with default value
(nop[char](), succeedWith(default))
], (_, s) => s)
full.map(s => %*{name: s})
proc mltOpt(name: string, opts: openArray[string]): OptionParser =
## match --<name>:<opt1>
## match --<name>:<opt1>,<opt2>
let copy = opts.toHashSet
let option = wordParser().validate(w => w in copy, fmt"invalid option for
{name}")
let comma = c(',')
let full = dash >> s(name) >> sep >> option.sepBy(comma)
full.map(s => %*{name: s})
proc intOpt(name: string, rng: HSlice[int, int]): OptionParser =
## match --<name>:<num>
let full = dash >> s(name) >> sep >> intParser().validate(n => n in rng,
fmt"expect {name} to be in range {rng}")
full.map(n => %*{name: n})
proc flgOpt(name: string, default: bool): OptionParser =
## match --<name>
## match --<name>:false
## match --<name>:true
## match --<name>:0
## match --<name>:1
let boolParser = oneOf(
s("true").result(true),
s("1").result(true),
s("false").result(false),
s("0").result(false)
).desc("expect flag in one of the following form: 0, 1, true, false")
let full = dash >> s(name) >> branch([
(sep, boolParser),
# last case always success with default value
(nop[char](), succeedWith(default))
], (_, b) => b)
full.map(b => %*{name: b})
proc mergeJson(js: seq[JsonNode]): JsonNode =
result = newJObject()
for j in js:
for k, v in j:
if k in result and v.kind == JArray:
result[k].add v
else:
result[k] = v
proc optionLineParser(opts: openArray[OptionParser]): OptionParser =
## <wsp> [ <opt> ( <wsp1> (<opt> | <eol>) )* ]
let opt = choice(opts)
let eol = eof.result(newJObject())
proc check(js: seq[JsonNode]): bool =
result = true
var ks: HashSet[string]
for j in js:
for k,v in j:
if k in ks and v.kind != JArray:
return false
ks.incl k
wsp >> (opt|eol).sepBy(whitespace).validate(check, "duplicated
option").map(mergeJson) << eof
proc mergeOpt(opts: varargs[seq[OptionParser]]): OptionParser =
var lis: seq[OptionParser]
for opt in opts: lis.add opt
optionLineParser(lis)
# -------------------------------------------------------------
# application
let commonOpt = @[
strOpt("command", ["copy", "edit"]),
strOpt("format", default="ogv", ["mp4", "ogv", "avi"]),
flgOpt("verbose", false),
intOpt("fps", 30..180),
mltOpt("feature", ["aa", "bb", "cc"]),
]
let encoder1SpecificOpt = @[
strOpt("option1", ["a1", "a2"]),
intOpt("quality", 1..3),
]
let encoder2SpecificOpt = @[
strOpt("option2", ["b1", "b2"]),
intOpt("quality", 1..10),
]
let encoder1OptionParser = wsp >> mergeOpt(commonOpt, encoder1SpecificOpt)
<< eof
let encoder2OptionParser = wsp >> mergeOpt(commonOpt, encoder2SpecificOpt)
<< eof
echo encoder1OptionParser.parse("")
echo encoder1OptionParser.parse(" ")
echo encoder1OptionParser.parse("--format")
echo encoder1OptionParser.parse("--format:mp4")
echo encoder1OptionParser.parse("--format=mp4")
echo encoder1OptionParser.parse("--format:mp4 --quality:1 --feature=aa,cc")
echo encoder2OptionParser.parse("--format:mp4 --quality:1 --feature=aa,cc
--verbose")
echo encoder1OptionParser.parse("--format:mp4 --quality:1 --feature=aa,cc
--verbose:1")
echo encoder1OptionParser.parse("--format:mp4 --quality:10") # invalid
range of speed for encoder1
echo encoder2OptionParser.parse("--format:mp4 --quality:10") # valid range
of speed for encoder2
echo encoder1OptionParser.parse("--format:mp4 --quality:1 --format:ogv") #
duplicated option
echo encoder2OptionParser.parse("--format:mp4 --quality:1 --option2=a1") #
invalid option for encode2
echo encoder2OptionParser.parse("--format:mp4 --quality:1 --option2=b1") #
valid option for encode2
#[
(kind: success, value: {}, tail: "", fromInput: "")
(kind: success, value: {}, tail: "", fromInput: " ")
(kind: success, value: {"format":"ogv"}, tail: "", fromInput: "--format")
(kind: success, value: {"format":"mp4"}, tail: "", fromInput:
"--format:mp4")
(kind: success, value: {"format":"mp4"}, tail: "", fromInput:
"--format=mp4")
(kind: success, value: {"format":"mp4","quality":1,"feature":["aa","cc"]},
tail: "", fromInput: "--format:mp4 --quality:1 --feature=aa,cc")
(kind: success, value:
{"format":"mp4","quality":1,"feature":["aa","cc"],"verbose":false}, tail: "",
fromInput: "--format:mp4 --quality:1 --feature=aa,cc --verbose")
(kind: success, value:
{"format":"mp4","quality":1,"feature":["aa","cc"],"verbose":true}, tail: "",
fromInput: "--format:mp4 --quality:1 --feature=aa,cc --verbose:1")
(kind: failure, expected: @["parse failure near `--quality:10`",
"\'format\'", "\'format\'", "\'verbose\'", "\'fps\'", "\'feature\'",
"\'option1\'", "expect quality to be in range 1 .. 3", "EOF"], tail:
"--format:mp4 --quality:10", fromInput: "--format:mp4 --quality:10")
(kind: success, value: {"format":"mp4","quality":10}, tail: "", fromInput:
"--format:mp4 --quality:10")
(kind: failure, expected: @["duplicated option"], tail: "--format:mp4
--quality:1 --format:ogv", fromInput: "--format:mp4 --quality:1 --format:ogv")
(kind: failure, expected: @["parse failure near `--option2=a1`",
"\'format\'", "\'format\'", "\'verbose\'", "\'fps\'", "\'feature\'", "invalid
option for option2", "\'quality\'", "EOF"], tail: "--format:mp4 --quality:1
--option2=a1", fromInput: "--format:mp4 --quality:1 --option2=a1")
(kind: success, value: {"format":"mp4","quality":1,"option2":"b1"}, tail:
"", fromInput: "--format:mp4 --quality:1 --option2=b1")
]#
Run
(A bit out of topic) After using honeycomb, my comment is that the library has
some fundamental problems. First, it define `type Parser[T] = proc(input:
string): ParseResult[T]`, the input string drop all contextual information like
position of line, I cannot generate more contextual error message from input.
Also, it force direct manipulation of string, this is easy to accidentally
create unnecessary strings. Secondly, it force user to eagerly generate error
string for failure. It is very common to backtrack, many error strings are just
created and drop away. A more decent implementation should be like
(Sorry, in typescript, I copy it from elsewhere)
export class Parser<T> {
run: (ctx: ParseContext) => ParseResult<T>;
}
export class ParseContext {
tokens: string;
ix = 0;
line = 0;
column = 0;
}
export class ParseResult<T> {
ctx: ParseContext;
ok: boolean;
value?: T;
genErrorTree?: () => ParseErrorTree;
}
export class ParseErrorTree {
name: string;
error: string;
children: ParseErrorTree[];
}
Run
Though parser combinator generally slower then hand-written procedural, it can
be optimized by writing low-level parser to close to the later given more
effort. A properly implemented parser is just a stationary graph of closures
that can be called many time without changes. Just-in-time like to inline them
(Not directly related Nim unless you compile to JS or llvm and run on llvm-jit)
and get closer to the version of procedural. Given the flexibility of PC, IMO,
it worth the penality.
In closing, if you do need to have a very strict validation, you will need the
full power of parser. It seems everyone has own flavour of doing parsing.
Anyway, honeycomb, IMO, is not 'real world' enough. I also checked another
library combparser, also have similar problem. The above code somehow works,
but need more polishing to fit your case. If I were you, I would roll out
yet-another-parser-combinator library. I am not persuading you to go this path.
It is simply because I know how to do it ideally (ideal in my mind). Anyway, I
guess I should stop here now. Good luck to your projects.
