Ottomata has uploaded a new change for review.
https://gerrit.wikimedia.org/r/324790
Change subject: Set api_version for eventlogging kafka consumers as well as
producers
......................................................................
Set api_version for eventlogging kafka consumers as well as producers
Bug: T142430
Change-Id: Iec32a4e1d6e9bfd8478e3ea4e59bf96d16971f70
---
M modules/role/manifests/eventlogging.pp
1 file changed, 25 insertions(+), 13 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/90/324790/1
diff --git a/modules/role/manifests/eventlogging.pp
b/modules/role/manifests/eventlogging.pp
index 843690b..84a95e7 100644
--- a/modules/role/manifests/eventlogging.pp
+++ b/modules/role/manifests/eventlogging.pp
@@ -150,6 +150,13 @@
['client-side-00', 'client-side-01']
)
+
+ # Append this to query params if set.
+ $kafka_api_version_param = $kafka_api_version ? {
+ undef => '',
+ default => "&api_version=${kafka_api_version}"
+ }
+
# Increase number and backoff time of retries for async
# analytics uses. If metadata changes, we should give
# more time to retry. NOTE: testing this in production
@@ -157,18 +164,11 @@
# out why and stop it. This either needs to be higher,
# or it is a bug in kafka-python.
# See: https://phabricator.wikimedia.org/T142430
- $kafka_retries_args = 'retries=6&retry_backoff_ms=200'
-
- # If $kafka_api_version is set (from hiera cluster config),
- # add it to kafka-python producer args.
- $kafka_producer_args = $kafka_api_version ? {
- undef => $kafka_retries_args,
- default => "api_version=${kafka_api_version}&${kafka_retries_args}"
- }
+ $kafka_producer_args =
"retries=6&retry_backoff_ms=200${kafka_api_version_param}"
eventlogging::service::processor { $client_side_processors:
format => '%q %{recvFrom}s %{seqId}d %t %o %{userAgent}i',
- input => $kafka_client_side_raw_uri,
+ input =>
"${kafka_client_side_raw_uri}${kafka_api_version_param}",
sid => $kafka_consumer_group,
outputs => [
"${kafka_schema_uri}&${kafka_producer_args}",
@@ -203,10 +203,16 @@
)
$kafka_consumer_group = 'eventlogging_consumer_mysql_00'
+ # Append this to query params if set.
+ $kafka_api_version_param = $kafka_api_version ? {
+ undef => '',
+ default => "&api_version=${kafka_api_version}"
+ }
+
# Kafka consumer group for this consumer is mysql-m4-master
eventlogging::service::consumer { $mysql_consumers:
- # auto commit offsets to kafka more often for mysql consumer:
- input => "${kafka_mixed_uri}&auto_commit_interval_ms=1000",
+ # auto commit offsets to kafka more often for mysql consumer
+ input =>
"${kafka_mixed_uri}&auto_commit_interval_ms=1000${$kafka_api_version_param}"
output =>
"mysql://${mysql_user}:${mysql_pass}@${mysql_db}?charset=utf8&statsd_host=${statsd_host}&replace=True",
sid => $kafka_consumer_group,
# Restrict permissions on this config file since it contains a
password.
@@ -275,9 +281,15 @@
'eventlogging_consumer_files_00'
)
+ # Append this to query params if set.
+ $kafka_api_version_param = $kafka_api_version ? {
+ undef => '',
+ default => "&api_version=${kafka_api_version}"
+ }
+
# Raw client side events:
eventlogging::service::consumer { 'client-side-events.log':
- input => "${kafka_client_side_raw_uri}&raw=True",
+ input =>
"${kafka_client_side_raw_uri}&raw=True${kafka_api_version_param}",
output => "file://${out_dir}/client-side-events.log",
sid => $kafka_consumer_group,
}
@@ -286,7 +298,7 @@
# 'blacklisted' during processing. Events are blacklisted
# from these logs for volume reasons.
eventlogging::service::consumer { 'all-events.log':
- input => $kafka_mixed_uri,
+ input => "${kafka_mixed_uri}${kafka_api_version_param}",
output => "file://${out_dir}/all-events.log",
sid => $kafka_consumer_group,
}
--
To view, visit https://gerrit.wikimedia.org/r/324790
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Iec32a4e1d6e9bfd8478e3ea4e59bf96d16971f70
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Ottomata <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
