So this is the full function. Note lines 4 and 5 - even if I define the 
"awards" and "idvs" arrays in the function, it doesn't work. If I define 
them outside the function, it works fine.

Thanks.

Brandon

function parsefeed(url,st,fin)

  db = SQLite.DB("/home/brandon/Documents/FPDS/fpds.db")
  schema = Data.Schema(["piid","task","mod","obl_amt"], [ASCIIString, 
ASCIIString, ASCIIString, Float64],)
  sink = SQLite.Sink(schema,db,"feeds")
  awards = ETree[]       #this doesn't matter
  idvs = ETree[]            #this doesn't matter

  for x in st:10:fin
    dfall = DataFrame([ASCIIString, ASCIIString, ASCIIString, Float64], 
[:piid, :task, :mod, :obl_amt], 0)
    df = DataFrame([ASCIIString, ASCIIString, ASCIIString, Float64], 
[:piid, :task, :mod, :obl_amt], 1)

    pg = xp_parse(readall(get(string(url,"&start=",x))))
    awards = xpath(pg,"/feed/entry/content/ns1:award")
    idvs = xpath(pg,"/feed/entry/content/ns1:IDV")

    if isdefined(:awards)
      for award in awards
        df[:piid] = 
string(LibExpat.find(award,"ns1:awardID/ns1:awardContractID/ns1:PIID#string"))
        df[:task] = string(LibExpat.find(award, 
"ns1:referencedIDVID/ns1:PIID#string"))
        df[:mod] = string(LibExpat.find(award, 
"ns1:awardID/ns1:awardContractID/ns1:modNumber#string"))
        df[:obl_amt] = 
float64(LibExpat.find(award,"ns1:dollarValues/ns1:obligatedAmount#string"))
        append!(dfall,df)
      end
    end

    if isdefined(:idvs)
      for idv in idvs
        df[:piid] = string(LibExpat.find(idv, 
"ns1:awardID/ns1:awardContractID/ns1:PIID#string"))
        df[:task] = ""
        df[:mod] = string(LibExpat.find(idv, 
"ns1:awardID/ns1:awardContractID/ns1:modNumber#string"))
        df[:obl_amt] = 
float64(LibExpat.find(award,"ns1:dollarValues/ns1:obligatedAmount#string"))
        append!(dfall,df)
      end
    end

    if size(dfall,1) > 0
      source = convertdf(dfall) #Data.Table(dfall) doesn't work, but I 
copied the function from DataStreams and called it convertdf and that 
solves the problem.
      Data.stream!(source,sink)
    end
  end
end

Reply via email to