OliverKeyes has uploaded a new change for review.
https://gerrit.wikimedia.org/r/249126
Change subject: Initial dashboard version for referred traffic
......................................................................
Initial dashboard version for referred traffic
Provides an initial version of the dashboard for digging into
search-engine-referred traffic.
Bug: T115652
Change-Id: Ia243accb579edc9e8c5b087b0ed7491a163fdd75
---
A .gitignore
A .gitreview
A functions.R
A server.R
A tab_documentation/traffic_byengine.md
A tab_documentation/traffic_summary.md
A ui.R
A wonderbolt.Rproj
A www/custom.js
A www/stylesheet.css
10 files changed, 207 insertions(+), 0 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/wikimedia/discovery/wonderbolt
refs/changes/26/249126/1
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..807ea25
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,3 @@
+.Rproj.user
+.Rhistory
+.RData
diff --git a/.gitreview b/.gitreview
new file mode 100644
index 0000000..be475f0
--- /dev/null
+++ b/.gitreview
@@ -0,0 +1,5 @@
+[gerrit]
+host=gerrit.wikimedia.org
+port=29418
+project=wikimedia/discovery/wonderbolt.git
+defaultbranch=master
diff --git a/functions.R b/functions.R
new file mode 100644
index 0000000..ef839ec
--- /dev/null
+++ b/functions.R
@@ -0,0 +1,42 @@
+library(polloi)
+library(data.table)
+
+# Custom function to allow for the selection betwene
+platform_select <- function(name){
+ return(selectizeInput(inputId = name, label = "Platform",
+ choices = c("All","Desktop","Mobile Web")))
+}
+
+# Read in the traffic data
+read_traffic <- function(){
+
+ # Read in the initial data and format.
+ data <- as.data.table(polloi::read_dataset(path =
"external_traffic/referer_data.tsv"))
+ data$is_search <- ifelse(data$is_search, "Referred by search", "Other")
+
+ # Write out the overall values for traffic
+ holding <- data[,j=list(pageviews = sum(pageviews)), by = c("timestamp",
"is_search", "access_method")]
+ holding <- split(holding, f = holding$access_method)
+ holding$all <- data[,j=list(pageviews = sum(pageviews)), by = c("timestamp",
"is_search")]
+ names(holding) <- c("Desktop", "Mobile Web", "All")
+ summary_traffic_data <<- lapply(holding, function(x){
+ return(reshape2::dcast(x, formula = timestamp ~ is_search, fun.aggregate =
sum))
+ })
+
+ # Generate per-engine values
+ holding <- data[, j = list(pageviews = sum(pageviews)), by = c("timestamp",
"search_engine", "access_method")]
+ holding <- split(holding, f = holding$access_method)
+ holding$all <- data[,j=list(pageviews = sum(pageviews)), by = c("timestamp",
"search_engine")]
+ names(holding) <- c("Desktop", "Mobile Web", "All")
+ bysearch_traffic_data <<- lapply(holding, function(x){
+ return(reshape2::dcast(x, formula = timestamp ~ search_engine,
fun.aggregate = sum))
+ })
+ return(invisible())
+}
+
+logscale <- function(data, logscale_setting){
+ if(logscale_setting){
+ return(cbind(data[,1], as.data.frame(apply(data[,2:ncol(data)], 2, log))))
+ }
+ return(data)
+}
diff --git a/server.R b/server.R
new file mode 100644
index 0000000..b16f226
--- /dev/null
+++ b/server.R
@@ -0,0 +1,23 @@
+source("functions.R")
+
+existing_date <- Sys.Date() - 1
+
+shinyServer(function(input, output){
+
+ if (Sys.Date() != existing_date) {
+ read_traffic()
+ existing_date <<- Sys.Date()
+ }
+
+ output$traffic_summary_dygraph <- renderDygraph({
+ polloi::make_dygraph(
+ data = summary_traffic_data[[input$platform_traffic_summary]],
+ xlab = "Date", ylab = "Pageviews", title = "Pageviews from external
search engines")
+ })
+
+ output$traffic_bysearch_dygraph <- renderDygraph({
+ polloi::make_dygraph(
+ data =
logscale(bysearch_traffic_data[[input$platform_traffic_bysearch]],
input$platform_traffic_bysearch_log),
+ xlab = "Date", ylab = "Pageviews", title = "Pageviews from external
search engines, broken down by engine")
+ })
+})
diff --git a/tab_documentation/traffic_byengine.md
b/tab_documentation/traffic_byengine.md
new file mode 100644
index 0000000..7191de5
--- /dev/null
+++ b/tab_documentation/traffic_byengine.md
@@ -0,0 +1,26 @@
+Traffic from external search engines, broken down
+=======
+
+A key metric in understanding the role external search engines play in
Wikipedia's readership and content discovery processes is a very direct one -
how many pageviews we get from them. This can be discovered very simply by
looking at our request logs.
+
+This dashboard simply breaks down the
[http://discovery.wmflabs.org/external/#traffic_summary](summary data) to
investigate how much traffic is coming from each search engine, individually.
As you can see, Google dominates, which is why we've included the option of
log-scaling
+the traffic.
+
+General trends
+------
+
+Outages and inaccuracies
+------
+None so far!
+
+Questions, bug reports, and feature suggestions
+------
+For technical, non-bug questions, [email
Mikhail](mailto:[email protected]?subject=Dashboard%20Question). If you
experience a bug or notice something wrong or have a suggestion, [open a ticket
in
Phabricator](https://phabricator.wikimedia.org/maniphest/task/create/?projects=Discovery)
in the Discovery board or [email
Dan](mailto:[email protected]?subject=Dashboard%20Question).
+
+<hr style="border-color: gray;">
+<p style="font-size: small; color: gray;">
+ <strong>Link to this dashboard:</strong>
+ <a href="http://discovery.wmflabs.org/external/#traffic_by_engine";>
+ http://discovery.wmflabs.org/external/#traffic_by_engine
+ </a>
+</p>
diff --git a/tab_documentation/traffic_summary.md
b/tab_documentation/traffic_summary.md
new file mode 100644
index 0000000..3954f81
--- /dev/null
+++ b/tab_documentation/traffic_summary.md
@@ -0,0 +1,26 @@
+Traffic from external search engines - summary
+=======
+
+A key metric in understanding the role external search engines play in
Wikipedia's readership and content discovery processes is a very direct one -
how many pageviews we get from them. This can be discovered very simply by
looking at our request logs.
+
+This dashboard simply looks at, very broadly, where our requests are coming
from - search engines or something else? It is split up into
+"all", "desktop" and "mobile web" platforms - but not apps, since the apps do
not log referers.
+
+General trends
+------
+
+Outages and inaccuracies
+------
+None so far!
+
+Questions, bug reports, and feature suggestions
+------
+For technical, non-bug questions, [email
Mikhail](mailto:[email protected]?subject=Dashboard%20Question). If you
experience a bug or notice something wrong or have a suggestion, [open a ticket
in
Phabricator](https://phabricator.wikimedia.org/maniphest/task/create/?projects=Discovery)
in the Discovery board or [email
Dan](mailto:[email protected]?subject=Dashboard%20Question).
+
+<hr style="border-color: gray;">
+<p style="font-size: small; color: gray;">
+ <strong>Link to this dashboard:</strong>
+ <a href="http://discovery.wmflabs.org/external/#traffic_summary";>
+ http://discovery.wmflabs.org/external/#traffic_summary
+ </a>
+</p>
diff --git a/ui.R b/ui.R
new file mode 100644
index 0000000..813523a
--- /dev/null
+++ b/ui.R
@@ -0,0 +1,39 @@
+library(shiny)
+library(shinydashboard)
+library(dygraphs)
+options(scipen = 500)
+source("functions.R")
+
+#Header elements for the visualisation
+header <- dashboardHeader(title = "External Search Traffic", disable = FALSE)
+
+sidebar <- dashboardSidebar(
+ tags$head(
+ tags$link(rel = "stylesheet", type = "text/css", href = "stylesheet.css"),
+ tags$script(src = "custom.js")
+ ),
+ sidebarMenu(menuItem(text = "Traffic"),
+ menuSubItem(text = "Summary", tabName = "traffic_summary"),
+ menuSubItem(text = "Pageviews by search engine", tabName =
"traffic_bysearch")
+ )
+)
+
+body <- dashboardBody(
+ tabItems(
+ tabItem(tabName = "traffic_summary",
+ platform_select("platform_traffic_summary"),
+ dygraphOutput("traffic_summary_dygraph"),
+ includeMarkdown("./tab_documentation/traffic_summary.md")
+ ),
+ tabItem(tabName = "traffic_bysearch",
+ platform_select("platform_traffic_bysearch"),
+ checkboxInput("platform_traffic_bysearch_log", label = "Log
scale", value = FALSE),
+ dygraphOutput("traffic_bysearch_dygraph"),
+ includeMarkdown("./tab_documentation/traffic_byengine.md")
+
+ )
+ )
+)
+
+dashboardPage(header, sidebar, body, skin = "black",
+ title = "External Search Dashboard | Discovery | Engineering |
Wikimedia Foundation")
diff --git a/wonderbolt.Rproj b/wonderbolt.Rproj
new file mode 100644
index 0000000..a213108
--- /dev/null
+++ b/wonderbolt.Rproj
@@ -0,0 +1,15 @@
+Version: 1.0
+
+RestoreWorkspace: Default
+SaveWorkspace: Default
+AlwaysSaveHistory: Default
+
+EnableCodeIndexing: Yes
+UseSpacesForTab: Yes
+NumSpacesForTab: 2
+Encoding: UTF-8
+
+RnwWeave: Sweave
+LaTeX: pdfLaTeX
+
+AutoAppendNewline: Yes
diff --git a/www/custom.js b/www/custom.js
new file mode 100644
index 0000000..7f65bc0
--- /dev/null
+++ b/www/custom.js
@@ -0,0 +1,25 @@
+$(function() {
+
+ // Enables linking to specific tabs:
+ if (window.location.hash){
+ var hash = $.trim(window.location.hash);
+ var tab = decodeURI(hash.substring(1, 100));
+ $('a[data-value=\"'+tab+'\"]').click();
+ }
+
+ // Enables clicking on a kpi summary value box to view the time series:
+ $('div[id^=kpi_summary_box_]').click(function(){
+ var parent_id = $(this).closest('div').attr('id');
+ var parent_target = parent_id.replace('_summary_box', '');
+ $('a[data-value=\"'+parent_target+'\"]').click();
+ });
+
+ // Visual feedback that the value box is now something you can click:
+ $('div[id^=kpi_summary_box_]').hover(function() {
+ $(this).css('cursor','pointer');
+ });
+
+ // Reveals the KPI dropdown menu at launch:
+ $('ul.sidebar-menu li.treeview').first().addClass('active');
+
+});
diff --git a/www/stylesheet.css b/www/stylesheet.css
new file mode 100644
index 0000000..2451b16
--- /dev/null
+++ b/www/stylesheet.css
@@ -0,0 +1,3 @@
+.kpi_date {
+ margin-top: 0px;
+}
--
To view, visit https://gerrit.wikimedia.org/r/249126
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ia243accb579edc9e8c5b087b0ed7491a163fdd75
Gerrit-PatchSet: 1
Gerrit-Project: wikimedia/discovery/wonderbolt
Gerrit-Branch: master
Gerrit-Owner: OliverKeyes <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
