Update 3: I make a script to automate the benchmarking to some degree, using
hyperfine
.nim
import std/[osproc,os],strformat,strutils,times,algorithm
let starttime = cpuTime()
type paramOpt = tuple[name,cmd: string]
let allparameters:seq[paramOpt]=
@[("danger","-d:danger"),("clang","--cc:clang"),("mas","--mm:markandsweep"),("threadsoff","--threads:off")]
var possibleCombos = newSeq[paramOpt]()
proc getpossibleCombos =
var base:paramOpt
let allparameterslen = allparameters.len
var basepoint,afterbase,length:int
proc appendTuple(b:seq[paramOpt]): paramOpt=
var a:paramOpt
for x in b:
a.name.add(x.name & " ")
a.cmd.add(x.cmd & " ")
return a
while length < allparameterslen-2:
afterbase = basepoint+length
while basepoint<allparameterslen-length:
for thisoption in allparameters[afterbase..<allparameterslen]:
possibleCombos.add((base.name & thisoption.name,base.cmd & thisoption.cmd))
base = appendTuple(allparameters[basepoint..basepoint+length])
inc afterbase
inc basepoint
basepoint = 0
inc length
base = appendTuple(allparameters[basepoint..basepoint+length])
inc basepoint
var lastitem:paramOpt
lastitem = appendTuple(allparameters)
possibleCombos.add lastitem
getpossibleCombos()
proc findmean(output: string): (float,string)=
# proc findmean(output: string): string=
var lines = output.splitLines(false)
for line in lines:
if line.contains("mean"):
var tmp1 = line.split(":")[1]
var tmp2 = tmp1.splitWhitespace()
var tmp3 = tmp2[1..4].join " "
# return tmp2.join " "
return (tmp2[0].parseFloat,tmp3)
proc sysexec(command: string): tuple=
var res = execCmdEx(command)
if res.exitCode != 0:
echo 1, " " & command
echo res.output
system.quit(1)
return res
# type result= tuple[time,name: string]
type result= tuple[time:(float,string),name: string]
var benchSlice: seq[result]
var benchdir = "bench" & format(times.now(),"yyyyMMdd-HHmm")
os.createDir(benchdir)
var sourcenim = os.paramStr(1)
var res = sysexec(fmt"nim --hints:off c -r --o:{benchdir}\base.exe
{sourcenim}")
echo "base: ",res.output
# var baseoutput = res.output
res = sysexec(fmt"hyperfine {benchdir}\base.exe")
benchSlice.add((res.output.findmean,"base.exe"))
# var params:string
# var warmup = if (os.paramCount() == 2): os.paramStr(2) else: 10.intToStr
for lang in ["c","cpp"]:
for param in possibleCombos:
var targetexe = "\"" & benchdir & "\\" & param.name & ".exe\""
res = sysexec(&"nim {lang} --hints:off -r {param.cmd} --o:{targetexe}
{sourcenim}")
echo param.cmd,": ",res.output
# system.quit(1)
res = sysexec(fmt"hyperfine -i --show-output {targetexe}")
# var res = sysexec(fmt"hyperfine {targetexe} --warmup {warmup}")
benchSlice.add((res.output.findmean,lang & " " & param.name))
benchSlice.sort(proc (a,b:result):int = cmp(a.time[0],b.time[0]))
let benchmarkfile = benchdir & "\\" & "benchmark.txt"
try: os.removeFile(benchmarkfile)
except: discard
var benchstr = benchSlice.join("\n")
echo benchstr
echo "Benchmarked ", possibleCombos.len * 2 + 1, " programs in ", cpuTime()
- starttime, "s"
writeFile(benchmarkfile,benchstr)
Run
Fibonacci 50 fares as follows:
(time: (92.3, "92.3 ms ± 4.2 ms"), name: "c danger clang mas")
(time: (101.0, "101.0 ms ± 7.2 ms"), name: "c danger clang mas threadsoff ")
(time: (105.9, "105.9 ms ± 11.3 ms"), name: "cpp danger clang mas")
(time: (107.9, "107.9 ms ± 7.4 ms"), name: "c clang mas")
(time: (109.6, "109.6 ms ± 9.6 ms"), name: "c clang mas threadsoff")
(time: (118.8, "118.8 ms ± 9.7 ms"), name: "cpp clang mas")
(time: (122.4, "122.4 ms ± 8.2 ms"), name: "cpp clang mas threadsoff")
(time: (123.9, "123.9 ms ± 10.1 ms"), name: "cpp danger clang mas
threadsoff ")
(time: (135.9, "135.9 ms ± 9.3 ms"), name: "c danger mas")
(time: (146.4, "146.4 ms ± 5.6 ms"), name: "cpp danger mas")
(time: (172.0, "172.0 ms ± 5.9 ms"), name: "c danger clang threadsoff")
(time: (173.5, "173.5 ms ± 6.9 ms"), name: "c danger threadsoff")
(time: (175.3, "175.3 ms ± 10.4 ms"), name: "c mas")
(time: (176.7, "176.7 ms ± 6.1 ms"), name: "cpp danger threadsoff")
(time: (178.9, "178.9 ms ± 11.4 ms"), name: "cpp danger clang threadsoff")
(time: (179.3, "179.3 ms ± 6.5 ms"), name: "cpp danger clang")
(time: (179.4, "179.4 ms ± 5.2 ms"), name: "c danger clang")
(time: (180.8, "180.8 ms ± 3.0 ms"), name: "c mas threadsoff")
(time: (181.5, "181.5 ms ± 4.6 ms"), name: "cpp danger")
(time: (181.6, "181.6 ms ± 7.4 ms"), name: "c danger")
(time: (184.8, "184.8 ms ± 6.2 ms"), name: "c clang threadsoff")
(time: (188.8, "188.8 ms ± 8.2 ms"), name: "c threadsoff")
(time: (188.9, "188.9 ms ± 10.7 ms"), name: "cpp clang threadsoff")
(time: (189.9, "189.9 ms ± 12.3 ms"), name: "cpp mas")
(time: (191.3, "191.3 ms ± 7.8 ms"), name: "cpp mas threadsoff")
(time: (191.4, "191.4 ms ± 5.6 ms"), name: "c clang")
(time: (194.6, "194.6 ms ± 5.5 ms"), name: "cpp clang")
(time: (209.6, "209.6 ms ± 3.0 ms"), name: "cpp threadsoff")
(time: (327.6, "327.6 ms ± 6.5 ms"), name: "base.exe")
Run
where mas is markandsweep
