Ppchelko has uploaded a new change for review. (
https://gerrit.wikimedia.org/r/372179 )
Change subject: WIP: Increase max kafka message size
......................................................................
WIP: Increase max kafka message size
Change-Id: Ice77027db74e67c519ce305e99ea22c3ff2c7b07
---
M hieradata/common.yaml
M modules/changeprop/manifests/init.pp
M modules/confluent/manifests/kafka/broker.pp
M modules/confluent/templates/kafka/server.properties.erb
4 files changed, 15 insertions(+), 1 deletion(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet
refs/changes/79/372179/1
diff --git a/hieradata/common.yaml b/hieradata/common.yaml
index 4922ec7..d6be87f 100644
--- a/hieradata/common.yaml
+++ b/hieradata/common.yaml
@@ -438,6 +438,7 @@
main-eqiad:
api_version: 0.9
zookeeper_cluster_name: main-eqiad
+ kafka_message_max_bytes: 2000000
brokers:
kafka1001.eqiad.wmnet:
id: 1001
@@ -449,6 +450,7 @@
main-codfw:
api_version: 0.9
zookeeper_cluster_name: main-codfw
+ kafka_message_max_bytes: 2000000
brokers:
kafka2001.codfw.wmnet:
id: 2001
diff --git a/modules/changeprop/manifests/init.pp
b/modules/changeprop/manifests/init.pp
index ef3b397..d83b107 100644
--- a/modules/changeprop/manifests/init.pp
+++ b/modules/changeprop/manifests/init.pp
@@ -48,7 +48,7 @@
include ::service::configuration
require ::changeprop::packages
-
+
service::node { 'changeprop':
enable => true,
port => 7272,
@@ -66,6 +66,7 @@
site => $::site,
redis_path => $redis_path,
redis_pass => $redis_pass,
+ kafka_max_bytes =>
$::kafka_clusters::main_eqiad::kafka_message_max_bytes,
},
auto_refresh => false,
init_restart => false,
diff --git a/modules/confluent/manifests/kafka/broker.pp
b/modules/confluent/manifests/kafka/broker.pp
index 99a7203..e2d3414 100644
--- a/modules/confluent/manifests/kafka/broker.pp
+++ b/modules/confluent/manifests/kafka/broker.pp
@@ -104,6 +104,9 @@
# partition in the fetch requests the replicas send to the leader.
# Default: undef (1048576)
#
+# [*kafka_message_max_bytes*]
+# The maximum size of a message that the server can receive. Default: undef
+#
# [*num_network_threads*]
# The number of threads handling network requests. Default: undef (3)
#
@@ -237,6 +240,7 @@
$replica_socket_receive_buffer_bytes = undef,
$num_replica_fetchers = 1,
$replica_fetch_max_bytes = undef,
+ $kafka_message_max_bytes = undef,
$num_network_threads = undef,
$num_io_threads = size($log_dirs),
diff --git a/modules/confluent/templates/kafka/server.properties.erb
b/modules/confluent/templates/kafka/server.properties.erb
index c2a8fe6..aec3cc3 100644
--- a/modules/confluent/templates/kafka/server.properties.erb
+++ b/modules/confluent/templates/kafka/server.properties.erb
@@ -273,3 +273,10 @@
# that additional support metrics will be collected and reported.
#
confluent.support.customer.id=anonymous
+
+<% if @kafka_message_max_bytes -%>
+
+# For the JobQueue we need larger messages then default
+message.max.bytes=<%= @kafka_message_max_bytes %>
+
+<% end -%>
\ No newline at end of file
--
To view, visit https://gerrit.wikimedia.org/r/372179
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ice77027db74e67c519ce305e99ea22c3ff2c7b07
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet
Gerrit-Branch: production
Gerrit-Owner: Ppchelko <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
