Ottomata has uploaded a new change for review.
https://gerrit.wikimedia.org/r/155549
Change subject: Add cron job to drop old data in HDFS
......................................................................
Add cron job to drop old data in HDFS
Change-Id: Ic917ccd027304f5a8a031bd20d8405f8fed30141
---
M manifests/role/analytics/refinery.pp
M manifests/site.pp
2 files changed, 4 insertions(+), 1 deletion(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/49/155549/1
diff --git a/manifests/role/analytics/refinery.pp
b/manifests/role/analytics/refinery.pp
index e544e40..40a916a 100644
--- a/manifests/role/analytics/refinery.pp
+++ b/manifests/role/analytics/refinery.pp
@@ -83,7 +83,7 @@
# keep this many days of data
$retention_days = 31
cron { 'refinery-drop-webrequest-partitions':
- command => "export
PYTHONPATH=\${PYTHONPATH}:${role::analytics::refinery::path}/python &&
${role::analytics::refinery::path}/bin/refinery-drop-webrequest-partitions -d
${retention_days} -D wmf >> ${log_file} 2>&1",
+ command => "export
PYTHONPATH=\${PYTHONPATH}:${role::analytics::refinery::path}/python &&
${role::analytics::refinery::path}/bin/refinery-drop-webrequest-partitions -d
${retention_days} -D wmf_raw -l /wmf/data/raw/webrequest >> ${log_file} 2>&1",
user => 'hdfs',
hour => '*/4',
}
diff --git a/manifests/site.pp b/manifests/site.pp
index 4340ab9..1d1181a 100644
--- a/manifests/site.pp
+++ b/manifests/site.pp
@@ -282,6 +282,9 @@
# HDFS from Kafka.
include role::analytics::refinery::camus
+ # Add cron job to delete old data in HDFS
+ include role::analytics::refinery::data::drop
+
# Oozie runs a monitor_done_flag job to make
# sure the _SUCCESS done-flag is written
# for each hourly webrequest import. This
--
To view, visit https://gerrit.wikimedia.org/r/155549
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ic917ccd027304f5a8a031bd20d8405f8fed30141
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Ottomata <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
