Christopher Johnson (WMDE) has submitted this change and it was merged.
Change subject: adds shell script for daily sparql data retrieval
......................................................................
adds shell script for daily sparql data retrieval
Change-Id: I6a56a21cb3c047f41cc452307a16afb73cea6c0a
---
A Rcron.sh
M bulk_sparql.R
2 files changed, 10 insertions(+), 8 deletions(-)
diff --git a/Rcron.sh b/Rcron.sh
new file mode 100755
index 0000000..00edb39
--- /dev/null
+++ b/Rcron.sh
@@ -0,0 +1,2 @@
+#! /bin/bash
+R CMD BATCH /srv/dashboards/shiny-server/wdm/bulk_sparql.R
diff --git a/bulk_sparql.R b/bulk_sparql.R
index d22dcac..482f4ce 100644
--- a/bulk_sparql.R
+++ b/bulk_sparql.R
@@ -1,8 +1,8 @@
#Bulk Query of WDQS and write to TSV
-source("config.R")
-
+source("/srv/dashboards/shiny-server/wdm/config.R")
output_path = "/srv/dashboards/shiny-server/wdm/data/sparql/"
-qlist <- read_file("./assets/rdfq.xml")
+qlist <- read_file("/srv/dashboards/shiny-server/wdm/assets/rdfq.xml")
+
rdfq <- xmlParse(qlist)
queries <- xmlToDataFrame(nodes = getNodeSet(rdfq, "//rdfq:select", c(rdfq =
"http://wikiba.se/rdfq#";)))
prefixes <- xmlToDataFrame(nodes = getNodeSet(rdfq, "//rdfq:prefix", c(rdfq =
"http://wikiba.se/rdfq#";)))
@@ -16,18 +16,18 @@
return(result)
}
-write_tsv <- function(x, filename){
+write_tsv <- function(result, filename){
date = Sys.Date()
file_uri <- paste0(output_path, filename)
- out = data.frame(date, x)
+ out = data.frame(date, result)
write.table(out, file=file_uri, append = TRUE, sep = "\t", row.names =
FALSE, col.names = FALSE)
}
bulk_sparql_query <- function(esc_queries) {
for(q in esc_queries) {
- x <- get_sparql_result(wdqs_uri, pfx, q)
- tsv_file <- paste0("spql", match(q, esc_queries), ".tsv")
- write_tsv(x, tsv_file)
+ result <- get_sparql_result(wdqs_uri, pfx, q)
+ tsv_filename <- paste0("spql", match(q, esc_queries), ".tsv")
+ write_tsv(result, tsv_filename)
}
}
--
To view, visit https://gerrit.wikimedia.org/r/247535
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I6a56a21cb3c047f41cc452307a16afb73cea6c0a
Gerrit-PatchSet: 2
Gerrit-Project: wikidata/analytics/dashboard
Gerrit-Branch: master
Gerrit-Owner: Christopher Johnson (WMDE) <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
