Christopher Johnson (WMDE) has uploaded a new change for review.
https://gerrit.wikimedia.org/r/238316
Change subject: first commit
......................................................................
first commit
Change-Id: I7d9abda4d64796bc749a1a33e47f65058bd3a5fd
---
A .gitignore
A README.md
A Wikidata community engagement tracking - Sheet1.ods
A assets/content/wikidata-edits.md
A assets/content/wikidata-pages.md
A assets/content/wikidata-properties.md
A assets/css/custom.css
A server.R
A ui.R
A utils.R
A wdm.Rproj
A wikidata-edits.tsv
A wikidata-pages.tsv
A wikidata-properties.tsv
A www/rainbow.js
A www/stylesheet.css
16 files changed, 516 insertions(+), 0 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/wikidata/analytics/dashboard
refs/changes/16/238316/1
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..aa7f856
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,3 @@
+.Rhistory
+.Rproj.user/
+
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..571fb08
--- /dev/null
+++ b/README.md
@@ -0,0 +1,19 @@
+## Quick start
+
+Install the dependencies:
+
+```
+$ R
+> install.packages(c("curl", "httpuv", "readr", "xts", "reshape2",
+ "RColorBrewer", "shiny", "shinydashboard", "dygraphs", "markdown",
+ "ggplot2", "toOrdinal", "plyr", "lubridate", "magrittr"))
+```
+
+Run the server:
+
+```
+$ R
+> library(shiny)
+> runApp(launch.browser = 0)
+```
+
diff --git a/Wikidata community engagement tracking - Sheet1.ods b/Wikidata
community engagement tracking - Sheet1.ods
new file mode 100644
index 0000000..19553bc
--- /dev/null
+++ b/Wikidata community engagement tracking - Sheet1.ods
Binary files differ
diff --git a/assets/content/wikidata-edits.md b/assets/content/wikidata-edits.md
new file mode 100644
index 0000000..6ca2876
--- /dev/null
+++ b/assets/content/wikidata-edits.md
@@ -0,0 +1,8 @@
+Wikidata Edits
+=======
+
+* Type: Aggregate
+* Source:
+* Update Interval:
+* Reliability:
+* Description:
diff --git a/assets/content/wikidata-pages.md b/assets/content/wikidata-pages.md
new file mode 100644
index 0000000..0e2f9e4
--- /dev/null
+++ b/assets/content/wikidata-pages.md
@@ -0,0 +1,8 @@
+Wikidata Pages
+=======
+
+* Type: Aggregate
+* Source:
+* Update Interval:
+* Reliability:
+* Description:
diff --git a/assets/content/wikidata-properties.md
b/assets/content/wikidata-properties.md
new file mode 100644
index 0000000..f6d317e
--- /dev/null
+++ b/assets/content/wikidata-properties.md
@@ -0,0 +1,8 @@
+Wikidata Properties
+=======
+
+* Type: Aggregate
+* Source:
+* Update Interval:
+* Reliability:
+* Description:
diff --git a/assets/css/custom.css b/assets/css/custom.css
new file mode 100644
index 0000000..fb0d105
--- /dev/null
+++ b/assets/css/custom.css
@@ -0,0 +1,3 @@
+.dygraph-legend {
+ background-color: #ECF0F5 !important;
+}
diff --git a/server.R b/server.R
new file mode 100644
index 0000000..65211da
--- /dev/null
+++ b/server.R
@@ -0,0 +1,46 @@
+## Version 0.2.0
+source("utils.R")
+
+existing_date <- (Sys.Date()-1)
+
+## Read in desktop data and generate means for the value boxes, along with a
time-series appropriate form for
+## dygraphs.
+read_desktop <- function(){
+ data <- download_set("wikidata-edits.tsv")
+ wikidata_edits <<- data
+
+ data <- download_set("wikidata-pages.tsv")
+ wikidata_pages <<- data
+
+ data <- download_set("wikidata-properties.tsv")
+ wikidata_properties <<- data
+
+ return(invisible())
+}
+
+shinyServer(function(input, output) {
+
+ if(Sys.Date() != existing_date){
+ read_desktop()
+ existing_date <<- Sys.Date()
+ }
+
+ output$wikidata_edits_plot <- renderDygraph({
+ smooth_level <- input$smoothing_wikidata_edits
+ make_dygraph(wikidata_edits,
+ "Date", "Edits", "Wikidata Edits, by month",
+ smoothing = ifelse(smooth_level == "global",
input$smoothing_global, smooth_level))
+ })
+ output$wikidata_pages_plot <- renderDygraph({
+ smooth_level <- input$smoothing_wikidata_pages
+ make_dygraph(wikidata_pages,
+ "Date", "Pages", "Wikidata Pages, by month", legend_name =
"pages",
+ smoothing = ifelse(smooth_level == "global",
input$smoothing_global, smooth_level))
+ })
+ output$wikidata_properties_plot <- renderDygraph({
+ smooth_level <- input$smoothing_wikidata_properties
+ make_dygraph(wikidata_properties,
+ "Date", "Properties", "Wikidata Properties, by month",
legend_name = "properties",
+ smoothing = ifelse(smooth_level == "global",
input$smoothing_global, smooth_level))
+ })
+})
diff --git a/ui.R b/ui.R
new file mode 100644
index 0000000..3af7b78
--- /dev/null
+++ b/ui.R
@@ -0,0 +1,52 @@
+library(shiny)
+library(shinydashboard)
+library(dygraphs)
+options(scipen = 500)
+
+#Header elements for the visualisation
+header <- dashboardHeader(title = "Wikidata Metrics", disable = FALSE)
+
+#Sidebar elements for the search visualisations.
+sidebar <- dashboardSidebar(
+ tags$head(
+ tags$link(rel = "stylesheet", type = "text/css", href = "stylesheet.css"),
+ tags$script(src = "rainbow.js")
+ ),
+ sidebarMenu(
+ menuItem(text = "Wikidata",
+ menuSubItem(text = "Edits", tabName = "wikidata_edits"),
+ menuSubItem(text = "Pages", tabName = "wikidata_pages"),
+ menuSubItem(text = "Properties", tabName =
"wikidata_properties")),
+ selectInput(inputId = "smoothing_global", label = "Smoothing (Global
Setting)", selectize = TRUE, selected = "day",
+ choices = c("No Smoothing" = "day", "Moving Average" =
"moving_avg",
+ "Weekly Median" = "week", "Monthly Median" =
"month"))
+ )
+)
+
+# Standardised input selector for smoothing
+smooth_select <- function(input_id, label = "Smoothing") {
+ return(selectInput(inputId = input_id, label = label, selectize = TRUE,
+ selected = "global", choices = c("Use Global Setting" =
"global",
+ "No Smoothing" = "day", "Moving Average" = "moving_avg",
+ "Weekly Median" = "week", "Monthly Median" = "month")))
+}
+
+#Body elements for the search visualisations.
+body <- dashboardBody(
+ tabItems(
+ tabItem(tabName = "wikidata_edits",
+ smooth_select("smoothing_wikidata_edits"),
+ dygraphOutput("wikidata_edits_plot"),
+ includeMarkdown("./assets/content/wikidata-edits.md")),
+ tabItem(tabName = "wikidata_pages",
+ smooth_select("smoothing_wikidata_pages"),
+ dygraphOutput("wikidata_pages_plot"),
+ includeMarkdown("./assets/content/wikidata-pages.md")),
+ tabItem(tabName = "wikidata_properties",
+ smooth_select("smoothing_wikidata_properties"),
+ dygraphOutput("wikidata_properties_plot"),
+ includeMarkdown("./assets/content/wikidata-properties.md"))
+ )
+)
+
+dashboardPage(header, sidebar, body, skin = "black")
diff --git a/utils.R b/utils.R
new file mode 100644
index 0000000..932daf6
--- /dev/null
+++ b/utils.R
@@ -0,0 +1,196 @@
+#Dependent libs
+library(plyr)
+library(readr)
+library(xts)
+library(reshape2)
+library(RColorBrewer)
+library(ggplot2)
+library(toOrdinal)
+library(lubridate)
+library(magrittr)
+
+#Utility functions for handling particularly common tasks
+download_set <- function(location){
+ location <- paste0("http://localhost/";, location,
+ "?ts=", gsub(x = Sys.time(), pattern = "(-| )",
replacement = ""))
+ con <- url(location)
+ return(readr::read_delim(con, delim = "\t"))
+}
+
+# Takes an untidy (read: dygraph-appropriate) dataset and adds
+# columns for each variable consisting of the smoothed, averaged mean
+smoother <- function(dataset, smooth_level = "day", rename = TRUE) {
+
+ # Determine the names and levels of aggregation. By default
+ # a smoothing level of "day" is assumed, which is no smoothing
+ # whatsoever, and so the original dataset is returned.
+ switch(smooth_level,
+ moving_avg = {
+ df <- apply(dataset[, -1, drop = FALSE], 2, function(x) {
+ y <- xts(x, dataset[, 1])
+ return(as.numeric(zoo::rollmean(x, k = 17, fill = NA)))
+ }) %>% as.data.frame %>% cbind(timestamp = dataset[, 1], .)
+ names(df) <- names(dataset)
+ if (rename) names(df)[-1] <- paste(names(df)[-1], " (Moving
average)")
+ return(df)
+ },
+ week = {
+ dataset$filter_1 <- lubridate::week(dataset[, 1])
+ dataset$filter_2 <- lubridate::year(dataset[, 1])
+ name_append <- ifelse(rename, " (Weekly average)", "")
+ },
+ month = {
+ dataset$filter_1 <- lubridate::month(dataset[, 1])
+ dataset$filter_2 <- lubridate::year(dataset[, 1])
+ name_append <- ifelse(rename, " (Monthly average)", "")
+ },
+ {
+ return(dataset)
+ }
+ )
+
+ # If we're still here it was weekly or monthly. Calculate
+ # the average for each unique permutation of filters
+
+ result <- ddply(.data = dataset,
+ .variables = c("filter_1", "filter_2"),
+ .fun = function(df, name_append){
+
+ # Construct output names for the averages, compute those
averages, and
+ # apply said names.
+ output_names <- paste0(names(df)[2:(ncol(df) - 2)],
name_append)
+ holding <- apply(df[, 2:(ncol(df) - 2), drop = FALSE], 2,
FUN = median) %>%
+ round %>% t %>% as.data.frame
+ names(holding) <- output_names
+
+ # Return the bound original values and averaged values
+ return(cbind(df[, 1, drop = FALSE], holding))
+ }, name_append = name_append)
+
+ return(result[, !(names(result) %in% c("filter_1","filter_2"))])
+}
+
+#Create a dygraph using our standard format.
+make_dygraph <- function(data, x, y, title, is_single = FALSE, legend_name =
NULL, use_si = TRUE, smoothing = "day") {
+ # cat("Making dygraph:", title, "\n"); # Debugging
+ if (is_single) {
+ data <- smoother(as.data.frame(data[, c(1, 3)]), smooth_level = smoothing)
+ data <- xts(data[, 2], data[, 1])
+ if (is.null(legend_name)) {
+ names(data) <- "events"
+ } else {
+ names(data) <- legend_name
+ }
+ } else {
+ data %<>% smoother(smooth_level = smoothing)
+ data <- xts(data[, -1], order.by = data[, 1])
+ }
+ return(dygraph(data, main = title, xlab = x, ylab = y) %>%
+ dyLegend(width = 400, show = "always") %>%
+ dyOptions(strokeWidth = 3,
+ colors = brewer.pal(max(3, ncol(data)), "Set2"),
+ drawPoints = FALSE, pointSize = 3, labelsKMB = use_si,
+ includeZero = TRUE) %>%
+ dyCSS(css = "./assets/css/custom.css"))
+}
+
+# Computes a median absolute deviation
+mad <- function(x) {
+ median(abs(x - median(x)))
+}
+
+compress <- function(x, round.by = 2) {
+ # by StackOverflow user 'BondedDust' : http://stackoverflow.com/a/28160474
+ div <- findInterval(as.numeric(gsub("\\,", "", x)),
+ c(1, 1e3, 1e6, 1e9, 1e12) )
+ paste(round( as.numeric(gsub("\\,","",x))/10^(3*(div-1)), round.by),
+ c("","K","M","B","T")[div], sep = "" )
+}
+
+# Conditional icon for widget.
+# Returns arrow-up icon on true (if true_direction is 'up'), e.g. load time %
change > 0
+cond_icon <- function(condition, true_direction = "up") {
+
+ if (true_direction == "up") {
+ return(icon(ifelse(condition, "arrow-up", "arrow-down")))
+ }
+
+ return(icon(ifelse(condition, "arrow-down", "arrow-up")))
+}
+
+# Conditional color for widget
+# Returns 'green' on true, 'red' on false, e.g. api usage % change > 0
+# load time % change < 0
+cond_color <- function(condition, true_color = "green") {
+ if(is.na(condition)){
+ return("black")
+ }
+
+ colours <- c("green","red")
+ return(ifelse(condition, true_color, colours[!colours == true_color]))
+}
+
+# Allows very quickly to get half a vector:
+half <- function(x, which = c("top", "bottom")) {
+ if (which == "top") {
+ return(head(x, n = length(x)/2))
+ }
+ return(tail(x, n = length(x)/2))
+}
+
+# Uses ggplot2 to create a pie chart in bar form. (Will look up actual name)
+gg_prop_bar <- function(data, cols) {
+ # `cols` = list(`item`, `prop`, `label`)
+ data$text_position <- cumsum(data[[cols$prop]]) + (c(0,
cumsum(data[[cols$prop]])[-nrow(data)]) - cumsum(data[[cols$prop]]))/2
+ ggplot(data, aes_string(x = 1, fill = cols$item)) +
+ geom_bar(aes_string(y = cols$prop), stat="identity") +
+ scale_fill_discrete(guide = FALSE, expand = c(0,0)) +
+ scale_y_continuous(expand = c(0,0)) +
+ scale_x_continuous(expand = c(0,0)) +
+ labs(x = NULL, y = NULL) +
+ coord_flip() +
+ theme_bw() +
+ theme(axis.ticks = element_blank(),
+ axis.text = element_blank(),
+ axis.title = element_blank(),
+ plot.margin = grid::unit(c(0, 0, -0.5, -0.5), "lines"),
+ panel.margin = grid::unit(0, "lines")) +
+ geom_text(aes_string(label = cols$label,
+ y = "text_position",
+ x = 1))
+}
+
+# Calculates percent change either in `x` or from `x` to `y`
+percent_change <- function(x, y = NULL) {
+ if(is.null(y)) {
+ return(100 * (x - c(NA, x[-length(x)])) / c(NA, x[-length(x)]))
+ }
+ return(100 * (y - x) / x)
+}
+
+# It's fairly common to need to grab the last N [whatever] of values.
+# The problem with using tail() for this comes in the case where (perhaps
+# due to backfilling) the last rows are not actually the last rows. This
+# function will provide a 'safe' tail mechanism.
+safe_tail <- function(x, n, silent = TRUE) {
+ if (!is.vector(x) && !is.data.frame(x)) {
+ stop("safe_trail() only works with vectors and data frames.")
+ }
+ # \code{silent} suppresses messages which may be used for debugging
+ if (is.vector(x)) {
+ return(tail(sort(x), n))
+ }
+ # Intelligently figure out which column is the date/timestamp column (in
case it's not the first column):
+ timestamp_column <- names(x)[sapply(x, class) %in% c("Date", "POSIXt",
"POSIXlt", "POSIXct")]
+ if (length(timestamp_column) == 0) {
+ if (!silent) {
+ message("No date/timestamp column detected for this dataset. It'd be
faster to use tail().")
+ }
+ return(tail(x, n))
+ }
+ if (length(timestamp_column) > 1) warning("More than one date/timestamp
column detected. Defaulting to the first one.")
+ if (!silent) {
+ message("Sorting by the date/timestamp column before returning the bottom
", n, " rows.")
+ }
+ return(tail(x[order(x[[timestamp_column[1]]]), ], n))
+}
diff --git a/wdm.Rproj b/wdm.Rproj
new file mode 100644
index 0000000..8e3c2eb
--- /dev/null
+++ b/wdm.Rproj
@@ -0,0 +1,13 @@
+Version: 1.0
+
+RestoreWorkspace: Default
+SaveWorkspace: Default
+AlwaysSaveHistory: Default
+
+EnableCodeIndexing: Yes
+UseSpacesForTab: Yes
+NumSpacesForTab: 2
+Encoding: UTF-8
+
+RnwWeave: Sweave
+LaTeX: pdfLaTeX
diff --git a/wikidata-edits.tsv b/wikidata-edits.tsv
new file mode 100644
index 0000000..526aba7
--- /dev/null
+++ b/wikidata-edits.tsv
@@ -0,0 +1,44 @@
+date edits
+2012-03-06 0
+2012-03-16 0
+2012-04-04 0
+2012-05-02 0
+2012-06-01 0
+2012-07-01 0
+2012-07-19 0
+2012-08-01 0
+2012-09-03 0
+2012-10-01 0
+2012-11-01 277282
+2012-12-01 739463
+2013-01-01 3004417
+2013-01-15 4394605
+2013-02-01 5622432
+2013-02-14 6571138
+2013-03-01 8566579
+2013-03-28 16230567
+2013-04-01 18200917
+2013-04-25 31046033
+2013-05-04 35037199
+2013-05-29 48726285
+2013-06-27 53560384
+2013-08-15 67289779
+2013-09-03 69920660
+2013-10-01 75856816
+2013-11-02 85554179
+2013-12-01 93577968
+2014-01-01 100445470
+2014-01-29 109197986
+2014-03-09 118653905
+2014-04-09 123785772
+2014-05-01 128627662
+2014-06-23 144505252
+2014-09-25 164090676
+2014-12-23 182571317
+2015-01-09 186439727
+2015-03-03 200530996
+2015-04-01 208039122
+2015-05-04 213859909
+2015-06-11 221731098
+2015-07-12 229631849
+2015-08-14 241198069
diff --git a/wikidata-pages.tsv b/wikidata-pages.tsv
new file mode 100644
index 0000000..eb3ccee
--- /dev/null
+++ b/wikidata-pages.tsv
@@ -0,0 +1,44 @@
+date wikidata pages
+2012-03-06 0
+2012-03-16 0
+2012-04-04 0
+2012-05-02 0
+2012-06-01 0
+2012-07-01 0
+2012-07-19 0
+2012-08-01 0
+2012-09-03 0
+2012-10-01 0
+2012-11-01 6291
+2012-12-01 234391
+2013-01-01 2112481
+2013-01-15 2541926
+2013-02-01 3753792
+2013-02-14 4323468
+2013-03-01 5357442
+2013-03-28 8132790
+2013-04-01 8739435
+2013-04-25 11153887
+2013-05-04 11868715
+2013-05-29 12575364
+2013-06-27 12673449
+2013-08-15 13613453
+2013-09-03 13785931
+2013-10-01 13992683
+2013-11-02 14065881
+2013-12-01 14154946
+2014-01-01 14364021
+2014-01-29 14443740
+2014-03-09 14634453
+2014-04-09 14934793
+2014-05-01 15342199
+2014-06-23 15722130
+2014-09-25 16320307
+2014-12-23 16871369
+2015-01-09 16917814
+2015-03-03 17624158
+2015-04-01 17952489
+2015-05-04 18056326
+2015-06-11 18284555
+2015-07-12 18846361
+2015-08-14 19000343
diff --git a/wikidata-properties.tsv b/wikidata-properties.tsv
new file mode 100644
index 0000000..898b7e7
--- /dev/null
+++ b/wikidata-properties.tsv
@@ -0,0 +1,44 @@
+date properties
+2012-03-06 0
+2012-03-16 0
+2012-04-04 0
+2012-05-02 0
+2012-06-01 0
+2012-07-01 0
+2012-07-19 0
+2012-08-01 0
+2012-09-03 0
+2012-10-01 0
+2012-11-01 0
+2012-12-01 0
+2013-01-01 0
+2013-01-15 0
+2013-02-01 0
+2013-02-14 0
+2013-03-01 127
+2013-03-28 255
+2013-04-01 261
+2013-04-25 328
+2013-05-04 345
+2013-05-29 411
+2013-06-27 486
+2013-08-15 629
+2013-09-03 656
+2013-10-01 740
+2013-11-02 793
+2013-12-01 825
+2014-01-01 840
+2014-01-29 855
+2014-03-09 962
+2014-04-09
+2014-05-01
+2014-06-23 1122
+2014-09-25 1196
+2014-12-23 1191
+2015-01-09 1188
+2015-03-03 1188
+2015-04-01 1188
+2015-05-04 1186
+2015-06-11 1186
+2015-07-12 1186
+2015-08-14 1210
diff --git a/www/rainbow.js b/www/rainbow.js
new file mode 100644
index 0000000..7f65bc0
--- /dev/null
+++ b/www/rainbow.js
@@ -0,0 +1,25 @@
+$(function() {
+
+ // Enables linking to specific tabs:
+ if (window.location.hash){
+ var hash = $.trim(window.location.hash);
+ var tab = decodeURI(hash.substring(1, 100));
+ $('a[data-value=\"'+tab+'\"]').click();
+ }
+
+ // Enables clicking on a kpi summary value box to view the time series:
+ $('div[id^=kpi_summary_box_]').click(function(){
+ var parent_id = $(this).closest('div').attr('id');
+ var parent_target = parent_id.replace('_summary_box', '');
+ $('a[data-value=\"'+parent_target+'\"]').click();
+ });
+
+ // Visual feedback that the value box is now something you can click:
+ $('div[id^=kpi_summary_box_]').hover(function() {
+ $(this).css('cursor','pointer');
+ });
+
+ // Reveals the KPI dropdown menu at launch:
+ $('ul.sidebar-menu li.treeview').first().addClass('active');
+
+});
diff --git a/www/stylesheet.css b/www/stylesheet.css
new file mode 100644
index 0000000..2451b16
--- /dev/null
+++ b/www/stylesheet.css
@@ -0,0 +1,3 @@
+.kpi_date {
+ margin-top: 0px;
+}
--
To view, visit https://gerrit.wikimedia.org/r/238316
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I7d9abda4d64796bc749a1a33e47f65058bd3a5fd
Gerrit-PatchSet: 1
Gerrit-Project: wikidata/analytics/dashboard
Gerrit-Branch: master
Gerrit-Owner: Christopher Johnson (WMDE) <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
