Bearloga has uploaded a new change for review.
https://gerrit.wikimedia.org/r/236055
Change subject: Fixed the WDQS usage aggregation query after having an
enlightening chat with WDQS's engineers.
......................................................................
Fixed the WDQS usage aggregation query after having an enlightening chat with
WDQS's engineers.
Change-Id: I87034c572ab1123b4af39f70119cf8ecf61a9185
---
M wdqs/basic_usage.R
1 file changed, 14 insertions(+), 18 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/wikimedia/discovery/golden
refs/changes/55/236055/1
diff --git a/wdqs/basic_usage.R b/wdqs/basic_usage.R
index 5dfffaa..9cd975e 100644
--- a/wdqs/basic_usage.R
+++ b/wdqs/basic_usage.R
@@ -22,18 +22,17 @@
# Write query and dump to file
query <- paste0("USE wmf;
- SELECT year, month, day,
- FIND_IN_SET(uri_path,
'/bigdata/namespace/wdq/sparql,/,/index.php') AS uri_path,
- IF(INSTR(uri_query, 'query') > 0, 'query', 'other') AS
uri_query,
- IF(INSTR(content_type, 'sparql-results') > 0, 'sparql
results', 'other') AS content_type,
+ SELECT year, month, day, uri_path,
+ (floor(cast(http_status as int) / 100) * 100) AS
http_category,
COUNT(*) AS n
FROM webrequest",
subquery,
- "AND webrequest_source = 'misc' AND FIND_IN_SET(http_status,
'200,301,302,303') > 0
- GROUP BY year, month, day,
- FIND_IN_SET(uri_path,
'/bigdata/namespace/wdq/sparql,/,/index.php'),
- IF(INSTR(uri_query, 'query') > 0, 'query', 'other'),
- IF(INSTR(content_type, 'sparql-results') > 0, 'sparql
results', 'other');")
+ "AND webrequest_source = 'misc'
+ AND uri_host = 'query.wikidata.org'
+ AND uri_path IN('/', '/bigdata/namespace/wdq/sparql')
+ GROUP BY year, month, day, uri_path,
+ floor(cast(http_status as int) / 100) * 100;")
+
query_dump <- tempfile()
cat(query, file = query_dump)
@@ -43,12 +42,9 @@
results <- read.delim(results_dump, sep = "\t", quote = "", as.is = TRUE,
header = TRUE)
file.remove(query_dump, results_dump)
- results$uri_path <- factor(results$uri_path, 0:3, c("other",
"/bigdata/namespace/wdq/sparql", "/", "/index.php"))
-
output <- data.frame(timestamp = as.Date(paste(results$year, results$month,
results$day, sep = "-")),
path = results$uri_path,
- query = results$uri_query,
- content = results$content_type,
+ http_status = results$http_category,
events = results$n,
stringsAsFactors = FALSE)
@@ -58,10 +54,10 @@
}
# Backlog (start date: 2015-07-28):
-# backlog <- function(days) {
-# for (i in days:1) try(main(Sys.Date() - i), silent = TRUE)
-# }; backlog(30) # as of 2015-08-27
+backlog <- function(days) {
+ for (i in days:1) try(main(Sys.Date() - i), silent = TRUE)
+}; backlog(30) # as of 2015-08-27In the
# Run and kill
-main()
-q(save = "no")
+# main()
+# q(save = "no")
--
To view, visit https://gerrit.wikimedia.org/r/236055
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I87034c572ab1123b4af39f70119cf8ecf61a9185
Gerrit-PatchSet: 1
Gerrit-Project: wikimedia/discovery/golden
Gerrit-Branch: master
Gerrit-Owner: Bearloga <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
