EBernhardson has uploaded a new change for review. (
https://gerrit.wikimedia.org/r/335140 )
Change subject: [WIP] Drop mediawiki logs in HDFS after 90 days
......................................................................
[WIP] Drop mediawiki logs in HDFS after 90 days
Change-Id: I5aceeb46fb5c7db0ed578b57463b276f39b4d059
---
M modules/role/manifests/analytics_cluster/refinery/data/drop.pp
1 file changed, 12 insertions(+), 1 deletion(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/40/335140/1
diff --git a/modules/role/manifests/analytics_cluster/refinery/data/drop.pp
b/modules/role/manifests/analytics_cluster/refinery/data/drop.pp
index 5c5ff07..eba3f34 100644
--- a/modules/role/manifests/analytics_cluster/refinery/data/drop.pp
+++ b/modules/role/manifests/analytics_cluster/refinery/data/drop.pp
@@ -7,6 +7,7 @@
$webrequest_log_file =
"${role::analytics_cluster::refinery::log_dir}/drop-webrequest-partitions.log"
$eventlogging_log_file =
"${role::analytics_cluster::refinery::log_dir}/drop-eventlogging-partitions.log"
+ $mediawiki_log_file =
"${role::analytics_cluster::refinery::log_dir}/drop-mediawiki-partitions.log"
# keep this many days of raw webrequest data
$raw_retention_days = 31
@@ -34,4 +35,14 @@
minute => '15',
hour => '*/4',
}
-}
\ No newline at end of file
+
+ $mediawiki_retention_days = 90
+ ['CirrusSearchRequestSet', 'ApiAction'].each |log_type| do
+ cron {"refinery-drop-${type}-partitions":
+ command => "export
PYTHONPATH=\${PYTHONPATH}:${role::analytics_cluster::refinery::path}/python &&
${role::analytics_cluster::refinery::path}/bin/refinery-drop-eventlogging-partitions
-d ${mediawiki_retention_days} -l
/wmf/data/raw/mediawiki/mediawiki_${log_type} >> ${mediawiki_log_file} 2>&1",
+ user => 'hdfs',
+ minute => '15',
+ hour => '*/4',
+ }
+ end
+}
--
To view, visit https://gerrit.wikimedia.org/r/335140
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I5aceeb46fb5c7db0ed578b57463b276f39b4d059
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: EBernhardson <ebernhard...@wikimedia.org>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
