Bearloga has uploaded a new change for review.
https://gerrit.wikimedia.org/r/235137
Change subject: Initial commit
......................................................................
Initial commit
Includes: script for acquiring aggregated data and storing it on the
server and a basic Shiny dashboard for displaying that aggregated data
Change-Id: Ibd72fa7676f4c0a46cfcd9d82716969f226171ae
---
A README.md
A assets/dataviz.css
A assets/wdqs_basic.md
A data_retrieval/wdqs.R
A server.R
A twilightsparql.Rproj
A ui.R
A utils.R
A www/custom.css
A www/custom.js
10 files changed, 261 insertions(+), 0 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/wikimedia/discovery/twilightsparql
refs/changes/37/235137/1
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..2a258f4
--- /dev/null
+++ b/README.md
@@ -0,0 +1,19 @@
+## Quick start
+
+Install the dependencies:
+
+```
+$ R
+> install.packages(c("curl", "httpuv", "readr", "xts", "reshape2",
+ "RColorBrewer", "shiny", "shinydashboard", "dygraphs", "markdown",
+ "ggplot2", "toOrdinal", "dplyr"))
+```
+
+Run the server:
+
+```
+$ R
+> library(shiny)
+> runApp(launch.browser = 0)
+```
+
diff --git a/assets/dataviz.css b/assets/dataviz.css
new file mode 100644
index 0000000..fb0d105
--- /dev/null
+++ b/assets/dataviz.css
@@ -0,0 +1,3 @@
+.dygraph-legend {
+ background-color: #ECF0F5 !important;
+}
diff --git a/assets/wdqs_basic.md b/assets/wdqs_basic.md
new file mode 100644
index 0000000..e9714d4
--- /dev/null
+++ b/assets/wdqs_basic.md
@@ -0,0 +1,23 @@
+Wikidata Query Service (WDQS) usage
+=======
+
+**Requests** in *Daily WDQS usage* are the non-query requests made to `/` and
`/index.php`.
+
+**Events** in *Daily SparQL usage* are the `sparql-results` requests.
+
+Outages and inaccuracies
+------
+
+ * None so far!
+
+Questions, bug reports, and feature suggestions
+------
+For technical, non-bug questions, [email
Mikhail](mailto:[email protected]?subject=Dashboard%20Question). If you
experience a bug or notice something wrong or have a suggestion, [open a ticket
in Phabricator](https://phabricator.wikimedia.org/maniphest/task/create/) in
the Discovery board or [email
Dan](mailto:[email protected]?subject=Dashboard%20Question).
+
+<hr style="border-color: gray;">
+<p style="font-size: small; color: gray;">
+ <strong>Link to this dashboard:</strong>
+ <a href="http://searchdata.wmflabs.org/metrics/#wdqs_usage";>
+ http://searchdata.wmflabs.org/metrics/#wdqs_usage
+ </a>
+</p>
diff --git a/data_retrieval/wdqs.R b/data_retrieval/wdqs.R
new file mode 100644
index 0000000..36bca99
--- /dev/null
+++ b/data_retrieval/wdqs.R
@@ -0,0 +1,86 @@
+#Config variables and setup
+options(scipen = 500,
+ q = "no")
+base_path <- "/a/aggregate-datasets/wdqs/"
+
+#Dependencies
+library(olivr)
+suppressPackageStartupMessages(library(data.table))
+
+if(!file.exists(base_path)){
+ dir.create(path = base_path)
+}
+
+#Conditional write; if the file exists, add x to the end. If it doesn't, write
an entirely new file.
+conditional_write <- function(x, file) {
+ if(file.exists(file)){
+ write.table(x, file, append = TRUE, sep = "\t", row.names = FALSE,
col.names = FALSE)
+ } else {
+ write.table(x, file, append = FALSE, sep = "\t", row.names = FALSE)
+ }
+}
+
+#Retrieves data for the WDQS stuff we care about, drops it in the
aggregate-datasets directory.
+#Should be run on stat1002, /not/ on the datavis machine.
+
+# Create a script that would produce raw data on usage of
+# - query.wikidata.org
+# - SPARQL endpoint: query.wikidata.org/bigdata/namespace/wdq/sparql
+
+#A function for creating a hive query aimed at the data from yesterday.
+# Central function
+main <- function(date = NULL) {
+
+ # Date handling
+ if(is.null(date)) {
+ date <- Sys.Date() - 1
+ }
+ subquery <- paste0(" WHERE year = ", lubridate::year(date),
+ " AND month = ", lubridate::month(date),
+ " AND day = ", lubridate::day(date), " ")
+
+ # Write query and dump to file
+ query <- paste0("USE wmf;
+ SELECT year, month, day,
+ FIND_IN_SET(uri_path,
'/bigdata/namespace/wdq/sparql,/,/index.php') AS uri_path,
+ IF(INSTR(uri_query, 'query') > 0, 'query', 'other') AS
uri_query,
+ IF(INSTR(content_type, 'sparql-results') > 0, 'sparql
results', 'other') AS content_type,
+ COUNT(*) AS n
+ FROM webrequest",
+ subquery,
+ "AND webrequest_source = 'misc' AND FIND_IN_SET(http_status,
'200,301,302,303') > 0
+ GROUP BY year, month, day,
+ FIND_IN_SET(uri_path,
'/bigdata/namespace/wdq/sparql,/,/index.php'),
+ IF(INSTR(uri_query, 'query') > 0, 'query', 'other'),
+ IF(INSTR(content_type, 'sparql-results') > 0, 'sparql
results', 'other');")
+ query_dump <- tempfile()
+ cat(query, file = query_dump)
+
+ # Query
+ results_dump <- tempfile()
+ system(paste0("export HADOOP_HEAPSIZE=1024 && hive -f ", query_dump, " > ",
results_dump))
+ results <- read.delim(results_dump, sep = "\t", quote = "", as.is = TRUE,
header = TRUE)
+ file.remove(query_dump, results_dump)
+
+ results$uri_path <- factor(results$uri_path, 0:3, c("other",
"/bigdata/namespace/wdq/sparql", "/", "/index.php"))
+
+ output <- data.frame(timestamp = as.Date(paste(results$year, results$month,
results$day, sep = "-")),
+ path = results$uri_path,
+ query = results$uri_query,
+ content = results$content_type,
+ events = results$n,
+ stringsAsFactors = FALSE)
+
+ # Write out
+ conditional_write(output, file.path(base_path, "wdqs_aggregates.tsv"))
+
+}
+
+# Backlog (start date: 2015-07-28):
+# backlog <- function(days) {
+# for (i in days:1) try(main(Sys.Date() - i), silent = TRUE)
+# }; backlog(30) # as of 2015-08-27
+
+#Run and kill
+main()
+q(save = "no")
diff --git a/server.R b/server.R
new file mode 100644
index 0000000..4beba98
--- /dev/null
+++ b/server.R
@@ -0,0 +1,47 @@
+source("utils.R")
+
+existing_date <- (Sys.Date() - 1)
+
+read_wdqs <- function(){
+ data <- download_set("wdqs_aggregates.tsv")
+ data <- data[order(data$timestamp),]
+ wdqs_usage <<- data
+ return(invisible())
+}
+
+shinyServer(function(input, output) {
+
+ if(Sys.Date() != existing_date){
+ read_wdqs()
+ existing_date <<- Sys.Date()
+ }
+
+ output$wdqs_usage_plot <- renderDygraph({
+ wdqs_usage %>%
+ dplyr::filter(path %in% c("/", "/index.php") & query == "other") %>%
+ dplyr::group_by(timestamp) %>%
+ summarise(total = sum(events)) %>%
+ # tidyr::spread(query, total) %>%
+ { xts(., order.by = .$timestamp) } %>%
+ dygraph(main = "Daily WDQS usage", group = "wdqs_basic",
+ xlab = "Date", ylab = "Requests") %>%
+ dyOptions(strokeWidth = 3, colors = brewer.pal(3, "Set2")[1],
+ drawPoints = TRUE, pointSize = 3, labelsKMB = TRUE,
+ includeZero = TRUE) %>%
+ dyCSS(css = "./assets/dataviz.css")
+ })
+
+ output$sparql_usage_plot <- renderDygraph({
+ wdqs_usage %>%
+ dplyr::filter(path == "/bigdata/namespace/wdq/sparql" & content ==
"sparql results") %>%
+ dplyr::select(c(timestamp, events)) %>%
+ { xts(., order.by = .$timestamp) } %>%
+ dygraph(main = "Daily SparkQL usage", group = "wdqs_basic",
+ xlab = "Date", ylab = "Events") %>%
+ dyOptions(strokeWidth = 3, colors = brewer.pal(3, "Set2")[2],
+ drawPoints = TRUE, pointSize = 3, labelsKMB = TRUE,
+ includeZero = TRUE) %>%
+ dyCSS(css = "./assets/dataviz.css")
+ })
+
+})
\ No newline at end of file
diff --git a/twilightsparql.Rproj b/twilightsparql.Rproj
new file mode 100644
index 0000000..d063e8b
--- /dev/null
+++ b/twilightsparql.Rproj
@@ -0,0 +1,13 @@
+Version: 1.0
+
+RestoreWorkspace: Default
+SaveWorkspace: Default
+AlwaysSaveHistory: Default
+
+EnableCodeIndexing: Yes
+UseSpacesForTab: Yes
+NumSpacesForTab: 2
+Encoding: UTF-8
+
+RnwWeave: knitr
+LaTeX: pdfLaTeX
diff --git a/ui.R b/ui.R
new file mode 100644
index 0000000..b69bb5f
--- /dev/null
+++ b/ui.R
@@ -0,0 +1,29 @@
+library(shiny)
+library(shinydashboard)
+library(dygraphs) # optional, used for dygraphs
+
+# Header elements for the visualization
+header <- dashboardHeader(title = "Wikidata Query Service", disable = FALSE)
+
+# Sidebar elements for the search visualizations
+sidebar <- dashboardSidebar(
+ tags$head(
+ tags$link(rel = "stylesheet", type = "text/css", href = "custom.css"),
+ tags$script(src = "custom.js")
+ ),
+ sidebarMenu(
+ menuItem(text = "WDQS Usage", tabName = "wdqs_usage")
+ ) # /sidebarMenu
+) # /dashboardSidebar
+
+#Body elements for the search visualizations.
+body <- dashboardBody(
+ tabItems(
+ tabItem(tabName = "wdqs_usage",
+ fluidRow(column(dygraphOutput("wdqs_usage_plot"), width = 6),
+ column(dygraphOutput("sparql_usage_plot"), width = 6)),
+ includeMarkdown("./assets/wdqs_basic.md"))
+ ) # /tabItems
+) # /dashboardBody
+
+dashboardPage(header, sidebar, body, skin = "purple")
\ No newline at end of file
diff --git a/utils.R b/utils.R
new file mode 100644
index 0000000..2d4f780
--- /dev/null
+++ b/utils.R
@@ -0,0 +1,18 @@
+#Dependent libs
+library(readr)
+library(xts)
+library(reshape2)
+library(RColorBrewer)
+
+#Utility functions for handling particularly common tasks
+download_set <- function(dataset){
+ con <- url(paste0("http://datasets.wikimedia.org/aggregate-datasets/wdqs/";,
dataset))
+ return(readr::read_delim(con, delim = "\t"))
+}
+
+# This function takes a number and returns a compressed string (e.g. 1624 =>
1.6K or 2K, depending on round.by)
+compress <- function(x, round.by = 2) {
+ # by StackOverflow user 'BondedDust' : http://stackoverflow.com/a/28160474
+ div <- findInterval(as.numeric(gsub("\\,", "", x)), c(1, 1e3, 1e6, 1e9,
1e12) )
+ paste(round( as.numeric(gsub("\\,","",x))/10^(3*(div-1)), round.by),
c("","K","M","B","T")[div], sep = "" )
+}
diff --git a/www/custom.css b/www/custom.css
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/www/custom.css
diff --git a/www/custom.js b/www/custom.js
new file mode 100644
index 0000000..4569dbe
--- /dev/null
+++ b/www/custom.js
@@ -0,0 +1,23 @@
+$(function() {
+
+ // Enables linking to specific tabs:
+ if (window.location.hash){
+ var hash = $.trim(window.location.hash);
+ var tab = decodeURI(hash.substring(1, 100));
+ $('a[data-value=\"'+tab+'\"]').click();
+ }
+ // Usage: append the tabName to the URL after the hash.
+
+ // Enables clicking on a kpi summary value box to view the time series:
+ $('div[id^=kpi_summary_box_]').click(function(){
+ var parent_id = $(this).closest('div').attr('id');
+ var parent_target = parent_id.replace('_summary_box', '');
+ $('a[data-value=\"'+parent_target+'\"]').click();
+ });
+
+ // Visual feedback that the value box is now something you can click:
+ $('div[id^=kpi_summary_box_]').hover(function() {
+ $(this).css('cursor','pointer');
+ });
+
+});
--
To view, visit https://gerrit.wikimedia.org/r/235137
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ibd72fa7676f4c0a46cfcd9d82716969f226171ae
Gerrit-PatchSet: 1
Gerrit-Project: wikimedia/discovery/twilightsparql
Gerrit-Branch: master
Gerrit-Owner: Bearloga <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
