EBernhardson has uploaded a new change for review. ( https://gerrit.wikimedia.org/r/335140 )
Change subject: [WIP] Drop mediawiki logs in HDFS after 90 days ...................................................................... [WIP] Drop mediawiki logs in HDFS after 90 days Change-Id: I5aceeb46fb5c7db0ed578b57463b276f39b4d059 --- M modules/role/manifests/analytics_cluster/refinery/data/drop.pp 1 file changed, 12 insertions(+), 1 deletion(-) git pull ssh://gerrit.wikimedia.org:29418/operations/puppet refs/changes/40/335140/1 diff --git a/modules/role/manifests/analytics_cluster/refinery/data/drop.pp b/modules/role/manifests/analytics_cluster/refinery/data/drop.pp index 5c5ff07..eba3f34 100644 --- a/modules/role/manifests/analytics_cluster/refinery/data/drop.pp +++ b/modules/role/manifests/analytics_cluster/refinery/data/drop.pp @@ -7,6 +7,7 @@ $webrequest_log_file = "${role::analytics_cluster::refinery::log_dir}/drop-webrequest-partitions.log" $eventlogging_log_file = "${role::analytics_cluster::refinery::log_dir}/drop-eventlogging-partitions.log" + $mediawiki_log_file = "${role::analytics_cluster::refinery::log_dir}/drop-mediawiki-partitions.log" # keep this many days of raw webrequest data $raw_retention_days = 31 @@ -34,4 +35,14 @@ minute => '15', hour => '*/4', } -} \ No newline at end of file + + $mediawiki_retention_days = 90 + ['CirrusSearchRequestSet', 'ApiAction'].each |log_type| do + cron {"refinery-drop-${type}-partitions": + command => "export PYTHONPATH=\${PYTHONPATH}:${role::analytics_cluster::refinery::path}/python && ${role::analytics_cluster::refinery::path}/bin/refinery-drop-eventlogging-partitions -d ${mediawiki_retention_days} -l /wmf/data/raw/mediawiki/mediawiki_${log_type} >> ${mediawiki_log_file} 2>&1", + user => 'hdfs', + minute => '15', + hour => '*/4', + } + end +} -- To view, visit https://gerrit.wikimedia.org/r/335140 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I5aceeb46fb5c7db0ed578b57463b276f39b4d059 Gerrit-PatchSet: 1 Gerrit-Project: operations/puppet Gerrit-Branch: production Gerrit-Owner: EBernhardson <ebernhard...@wikimedia.org> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits