So this is the full function. Note lines 4 and 5 - even if I define the
"awards" and "idvs" arrays in the function, it doesn't work. If I define
them outside the function, it works fine.
Thanks.
Brandon
function parsefeed(url,st,fin)
db = SQLite.DB("/home/brandon/Documents/FPDS/fpds.db")
schema = Data.Schema(["piid","task","mod","obl_amt"], [ASCIIString,
ASCIIString, ASCIIString, Float64],)
sink = SQLite.Sink(schema,db,"feeds")
awards = ETree[] #this doesn't matter
idvs = ETree[] #this doesn't matter
for x in st:10:fin
dfall = DataFrame([ASCIIString, ASCIIString, ASCIIString, Float64],
[:piid, :task, :mod, :obl_amt], 0)
df = DataFrame([ASCIIString, ASCIIString, ASCIIString, Float64],
[:piid, :task, :mod, :obl_amt], 1)
pg = xp_parse(readall(get(string(url,"&start=",x))))
awards = xpath(pg,"/feed/entry/content/ns1:award")
idvs = xpath(pg,"/feed/entry/content/ns1:IDV")
if isdefined(:awards)
for award in awards
df[:piid] =
string(LibExpat.find(award,"ns1:awardID/ns1:awardContractID/ns1:PIID#string"))
df[:task] = string(LibExpat.find(award,
"ns1:referencedIDVID/ns1:PIID#string"))
df[:mod] = string(LibExpat.find(award,
"ns1:awardID/ns1:awardContractID/ns1:modNumber#string"))
df[:obl_amt] =
float64(LibExpat.find(award,"ns1:dollarValues/ns1:obligatedAmount#string"))
append!(dfall,df)
end
end
if isdefined(:idvs)
for idv in idvs
df[:piid] = string(LibExpat.find(idv,
"ns1:awardID/ns1:awardContractID/ns1:PIID#string"))
df[:task] = ""
df[:mod] = string(LibExpat.find(idv,
"ns1:awardID/ns1:awardContractID/ns1:modNumber#string"))
df[:obl_amt] =
float64(LibExpat.find(award,"ns1:dollarValues/ns1:obligatedAmount#string"))
append!(dfall,df)
end
end
if size(dfall,1) > 0
source = convertdf(dfall) #Data.Table(dfall) doesn't work, but I
copied the function from DataStreams and called it convertdf and that
solves the problem.
Data.stream!(source,sink)
end
end
end