Hello,

I am trying to build a list of mirrors and append the date when the mirror has 
been refreshed. (For doing that I take the last-modified header of a file)

The program built without errors. The program begin to parse and print but, it 
ends if it meet an error during the parse.

Is there a way to "bypass" the crap mirror and go to the next one ?

Here is the error output : 
    
    
    https://mirror.labkom.id/pub/OpenBSD
    2020-05-27 15:45
    2020-05-27 20:08
    ...
    ...
    https://mirrors.nav.ro/pub/OpenBSD
    2020-05-27 15:45
    2020-05-26 20:01
    /home/f/code/mirrosnap/mirrosnap.nim(55) mirrosnap
    /home/f/code/mirrosnap/mirrosnap.nim(44) makeDalist
    /usr/local/lib/nim/pure/httpclient.nim(254) lastModified
    /usr/local/lib/nim/pure/times.nim(2436) parse
    /usr/local/lib/nim/pure/times.nim(2412) parse
    /usr/local/lib/nim/pure/times.nim(1827) raiseParseException
    Error: unhandled exception: Failed to parse '' with format 'ddd, dd MMM 
yyyy HH:mm:ss 'GMT''. Parsing ended but there was still patterns remaining 
[TimeParseError]
    
    
    Run

Here is the code : 
    
    
    import httpClient
    import strutils
    import re
    import tables
    #import osproc
    import times
    
    var
      client = newHttpClient(timeout = 20000)
      mirloc = initTable[string, string]() # mirrors table
      basemod = initTable[string, string]() # table : url + base freshness
      pkgmod = initTable[string, string]() # table : url + packages freshness
      kbase: string
      kpkg: string
      basem: Response
      pkgm: Response
    
    let
      httpslist: string = 
"https://cvsweb.openbsd.org/cgi-bin/cvsweb/~checkout~/www/httpslist";
      #(myArchnl, errarch) = execCmdEx("uname -m")
      #myArch = myArchnl.replace(re"\n")
    
    proc mirrorsDico(link: string) =
      let
        contents = client.getContent(link)
        mirrors = contents.splitLines()
      for m in mirrors:
        if m =~ re"(^http.*(BSD|bsd))(\s{2,})(\w.*$)":
          # mirror url ------- location
          mirloc[matches[0]] = matches[3]
    
    mirrorsDico(httpslist)
    
    proc makeDalist(arch: string) =
      let
        base = "/snapshots/" & $arch & "/SHA256"
        pkg = "/snapshots/packages/" & $arch & "/SHA256"
      for k, v in mirloc:
        kbase = k & base
        kpkg = k & pkg
        basem = client.get(kbase)
        pkgm = client.get(kpkg)
        let
          bma = basem.lastModified()
          pma = pkgm.lastModified()
          bm = bma.format("yyyy-MM-dd HH:mm")
          pm = pma.format("yyyy-MM-dd HH:mm")
        #basemod[k] = $bm
        #pkgmod[k] = $pm
    # TODO find a way to handle error and bypass it
        echo k
        echo bm
        echo pm
    
    makeDalist("amd64")
    
    #for mir, bf in basemod:
    #  echo(mir, " | ", bf)
    
    
    Run

Don't hesitate to criticize, this is my first try with nim ;)

Regards.

fredg.

Reply via email to