OliverKeyes has uploaded a new change for review.
https://gerrit.wikimedia.org/r/248115
Change subject: Add referer-tracking scripts.
......................................................................
Add referer-tracking scripts.
What it says on the tin!
Bug: T116295
Change-Id: I2e11d419321bb0f55f614c75b28267eee1b6d174
---
M config.R
A external/search_referers.R
2 files changed, 46 insertions(+), 1 deletion(-)
git pull ssh://gerrit.wikimedia.org:29418/wikimedia/discovery/golden
refs/changes/15/248115/1
diff --git a/config.R b/config.R
index 1b50863..79764f2 100644
--- a/config.R
+++ b/config.R
@@ -3,7 +3,7 @@
# Core paths
write_root <- "/a/aggregate-datasets/"
-dirs <- c("maps","wdqs", "search")
+dirs <- c("maps","wdqs", "search", "external_traffic")
write_dirs <- paste0(write_root, dirs)
# Dependencies
diff --git a/external/search_referers.R b/external/search_referers.R
new file mode 100644
index 0000000..8edb06c
--- /dev/null
+++ b/external/search_referers.R
@@ -0,0 +1,45 @@
+# Per-file config:
+base_path <- paste0(write_root, "external_traffic/")
+check_dir(base_path)
+
+main <- function(date = NULL){
+
+ # Date handling
+ if(is.null(date)){
+ date <- Sys.Date() - 1
+ }
+
+ # Date subquery
+ subquery <- date_clause(date)
+
+ # Write query and run it
+ query <- paste0("ADD JAR /home/ironholds/refinery-hive-0.0.21-SNAPSHOT.jar;
+ CREATE TEMPORARY FUNCTION is_external_search AS
+ 'org.wikimedia.analytics.refinery.hive.IsExternalSearchUDF';
+ CREATE TEMPORARY FUNCTION classify_referer AS
+ 'org.wikimedia.analytics.refinery.hive.RefererClassifyUDF';
+ CREATE TEMPORARY FUNCTION get_engine AS
+
'org.wikimedia.analytics.refinery.hive.IdentifySearchEngineUDF';
+ USE wmf;
+ SELECT year, month, day,
+ is_external_search(referer) AS is_search,
+ classify_referer(referer) AS referer_class,
+ get_engine(referer) as search_engine,
+ COUNT(*) AS pageviews
+ FROM webrequest
+ ", subquery,
+ "AND webrequest_source IN('text','mobile') AND is_pageview =
true
+ GROUP BY year, month, day, is_external_search(referer),
classify_referer(referer),
+ get_engine(referer);")
+ results <- query_hive(query)
+
+ # Sanitise the resulting data
+ results <- results[!is.na(results$month),]
+ results$timestamp <- as.Date(paste(results$year, results$month, results$day,
sep = "-"))
+ results <- as.data.table(results[, c("timestamp", "is_search",
"referer_class", "search_engine", "pageviews")])
+ results$is_search <- ifelse(results$is_search == "true", TRUE, FALSE)
+
+ # Write out
+ conditional_write(results, file.path(base_path, "referer_data.tsv"))
+}
+
--
To view, visit https://gerrit.wikimedia.org/r/248115
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I2e11d419321bb0f55f614c75b28267eee1b6d174
Gerrit-PatchSet: 1
Gerrit-Project: wikimedia/discovery/golden
Gerrit-Branch: master
Gerrit-Owner: OliverKeyes <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
