Ottomata has uploaded a new change for review.
https://gerrit.wikimedia.org/r/269759
Change subject: Add $check_jar and $camus_jar parameterize to camus::job
......................................................................
Add $check_jar and $camus_jar parameterize to camus::job
This allows us to hardcode specific releases of dependencies
when camus is launched at runtime. This will avoid unintentional
bugs in new releases from unexpectedly affecting working jobs.
Change-Id: Ic86284fb3da5f7ebdedbadee224bfab2cbdbc036
---
M manifests/role/analytics/refinery.pp
M modules/camus/manifests/job.pp
2 files changed, 34 insertions(+), 5 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/59/269759/1
diff --git a/manifests/role/analytics/refinery.pp
b/manifests/role/analytics/refinery.pp
index 69b31ae..37dddcf 100644
--- a/manifests/role/analytics/refinery.pp
+++ b/manifests/role/analytics/refinery.pp
@@ -57,9 +57,15 @@
require role::analytics::refinery
include role::kafka::analytics::config
- # Make all uses of camus::job set kafka_brokers to this
+ # Make all uses of camus::job set default kafka_brokers and camus_jar.
+ # If you build a new camus or refinery, and you want to use it, you'll
+ # need to change these. You can also override these defaults
+ # for a particular camus::job instance by setting the parameter on
+ # the camus::job declaration.
Camus::Job {
- kafka_brokers =>
suffix($role::kafka::analytics::config::brokers_array, ':9092')
+ kafka_brokers =>
suffix($role::kafka::analytics::config::brokers_array, ':9092'),
+ camus_jar =>
"${role::analytics::refinery::path}/org/wikimedia/analytics/camus-wmf/camus-wmf-0.1.0-wmf6.jar",
+ check_jar =>
"${role::analytics::refinery::path}/org/wikimedia/analytics/refinery/refinery-job-0.0.26.jar",
}
# Import webrequest_* topics into /wmf/data/raw/webrequest
diff --git a/modules/camus/manifests/job.pp b/modules/camus/manifests/job.pp
index f31bd4f..0a0a6f4 100644
--- a/modules/camus/manifests/job.pp
+++ b/modules/camus/manifests/job.pp
@@ -18,6 +18,19 @@
# [*user*]
# The camus cron will be run by this user.
#
+# [*camus_jar*]
+# Path to camus.jar. Default undef,
+# (/srv/deployment/analytics/refinery/artifacts/camus-wmf.jar)
+#
+# [*check*]
+# If true, CamusPartitionChecker will be run after the Camus run finishes.
+# Default: undef, (false)
+#
+# [*check_jar*]
+# Path to jar with CamusPartitionChecker. This is ignored if
+# $check is false. Default: undef,
+# (/srv/deployment/analytics/refinery/artifacts/refinery-job.jar)
+#
# [*libjars*]
# Any additional jar files to pass to Hadoop when starting the MapReduce
job.
#
@@ -39,8 +52,10 @@
$kafka_brokers,
$script = '/srv/deployment/analytics/refinery/bin/camus',
$user = 'hdfs',
- $libjars = undef,
+ $camus_jar = undef,
$check = undef,
+ $check_jar = undef,
+ $libjars = undef,
$template = "camus/${title}.erb",
$template_variables = {},
$hour = undef,
@@ -59,6 +74,11 @@
content => template($template),
}
+ $camus_jar_opt = $camus_jar ? {
+ undef => '',
+ default => "--jar ${camus_jar}",
+ }
+
$libjars_opt = $libjars ? {
undef => '',
default => "--libjars ${libjars}",
@@ -66,10 +86,13 @@
$check_opt = $check ? {
undef => '',
- default => '--check',
+ default => $check_jar ? {
+ undef => '--check',
+ default => "--check --check-jar ${check_jar}",
+ }
}
- $command = "${script} --run --job-name camus-${title} ${libjars_opt}
${check_opt} ${properties_file} >> ${log_file} 2>&1"
+ $command = "${script} --run --job-name camus-${title} ${camus_jar_opt}
${libjars_opt} ${check_opt} ${properties_file} >> ${log_file} 2>&1"
cron { "camus-${title}":
command => $command,
--
To view, visit https://gerrit.wikimedia.org/r/269759
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ic86284fb3da5f7ebdedbadee224bfab2cbdbc036
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Ottomata <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
