Ottomata has submitted this change and it was merged.
Change subject: Add legacy_tsv dataset to webrequest status dump script
......................................................................
Add legacy_tsv dataset to webrequest status dump script
Change-Id: Iafb708bfe0cd4e39bc0a36449b60ab7f04131bb6
---
M bin/refinery-dump-status-webrequest-partitions
1 file changed, 28 insertions(+), 0 deletions(-)
Approvals:
Ottomata: Verified; Looks good to me, approved
diff --git a/bin/refinery-dump-status-webrequest-partitions
b/bin/refinery-dump-status-webrequest-partitions
index a72937c..150b045 100755
--- a/bin/refinery-dump-status-webrequest-partitions
+++ b/bin/refinery-dump-status-webrequest-partitions
@@ -17,6 +17,7 @@
--datasets DATASET1,DATASET2,...
-- Select the datasets to output data for.
The following datasets are available:
+ legacy_tsvs -- webrequest (refined tables)
(daily)
pagecounts-all-sites -- pagecounts-all-sites (hourly)
pagecounts-raw -- pagecounts-raw (hourly)
raw_webrequest -- Raw webrequest (hourly)
@@ -68,6 +69,7 @@
DATASET_VISIBILITIES["$DATASET"]=no
}
+add_dataset "legacy_tsvs" "daily" " 5xx | api | edits |
glam_nara | mobile | sampled | zero |"
add_dataset "pagecounts_all_sites" "hourly" " file name date | page |
project |"
add_dataset "pagecounts_raw" "hourly" " file name date | page | project |"
add_dataset "raw_webrequest" "hourly" " bits | misc | mobile | text |
upload |"
@@ -323,6 +325,32 @@
done
}
+dump_dataset_legacy_tsvs() {
+ local DATE="$1"
+
+ local SUFFIX=".tsv.log-$(date --utc -d "$DATE 1 hour" +"%Y%m%d").gz"
+
+ local BASE
+ for BASE in \
+ 5xx/5xx \
+ api/api-usage \
+ edits/edits \
+ glam_nara/glam_nara \
+ mobile/mobile-sampled-100 \
+ sampled/sampled-1000 \
+ zero/zero \
+
+ do
+ local STATUS="X"
+ local FILE_ABS="$ARCHIVE_DATA_DIR_ABS/webrequest/$BASE$SUFFIX"
+ if [ -e "$FILE_ABS" ]
+ then
+ STATUS="."
+ fi
+ log_no_lf " $STATUS |"
+ done
+}
+
dump_dataset_pagecounts_file() {
local DATASET="$1"
local KIND="$2"
--
To view, visit https://gerrit.wikimedia.org/r/188007
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Iafb708bfe0cd4e39bc0a36449b60ab7f04131bb6
Gerrit-PatchSet: 3
Gerrit-Project: analytics/refinery
Gerrit-Branch: master
Gerrit-Owner: QChris <[email protected]>
Gerrit-Reviewer: Ottomata <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
