Ottomata has uploaded a new change for review.
https://gerrit.wikimedia.org/r/91664
Change subject: Updating with recent upstream changes to varnishkafka.conf
......................................................................
Updating with recent upstream changes to varnishkafka.conf
Change-Id: Id856d1f9a2cbc1bb6f22dc7bd95c2df950a10976
---
M manifests/defaults.pp
M manifests/init.pp
M templates/varnishkafka.conf.erb
3 files changed, 72 insertions(+), 25 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet/varnishkafka
refs/changes/64/91664/1
diff --git a/manifests/defaults.pp b/manifests/defaults.pp
index 36e975b..e23e4ea 100644
--- a/manifests/defaults.pp
+++ b/manifests/defaults.pp
@@ -7,9 +7,9 @@
$output = 'kafka'
$format_type = 'string'
- $format = '%l %n %t
%{Varnish:time_firstbyte}x %h %{Varnish:handling}x/%s %b %m
http://%{Host}i%U%q - %{Content-Type}o %{Referer}i
%{X-Forwarded-For}i %{User-agent!escape}i'
+ $format = '%l %n %t
%{Varnish:time_firstbyte}x %h %{Varnish:handling}x/%s %b %m
http://%{Host}i%U%q - %{Content-Type}o %{Referer}i
%{X-Forwarded-For}i %{User-agent!escape}i %{Accept-Language}i'
- $format_key_type = 'kafka'
+ $format_key_type = 'string'
$format_key = undef
$partition = -1
@@ -17,6 +17,7 @@
$message_send_max_retries = 3
$topic_request_required_acks = 1
$topic_message_timeout_ms = 60000
+ $compression_codec = 'none'
$varnish_opts = {
'm' => 'RxRequest:^(?!PURGE$)',
@@ -24,10 +25,8 @@
$log_data_copy = true
$log_level = 6
- $log_stderr = true
+ $log_stderr = false
$log_syslog = true
-
- $daemon_opts = undef
$conf_template = 'varnishkafka/varnishkafka.conf.erb'
$default_template = 'varnishkafka/varnishkafka.default.erb'
diff --git a/manifests/init.pp b/manifests/init.pp
index 3a39eae..5c6edb2 100644
--- a/manifests/init.pp
+++ b/manifests/init.pp
@@ -30,6 +30,9 @@
# $topic_request_required_acks - Required ack level. Default: 1
# $topic_message_timeout_ms - Local message timeout (milliseconds).
# Default: 60000
+# $compression_codec - Compression codec to use when sending
batched messages
+# Kafka. Valid values are 'none', 'gzip',
and 'snappy'.
+# Default: none
# $varnish_opts - Arbitrary hash of varnish CLI options.
# Default: { 'm' =>
'RxRequest:^(?!PURGE$)' }
# $log_data_copy - If true, log tag data read from VSL files
@@ -58,6 +61,7 @@
$message_send_max_retries =
$varnishkafka::defaults::message_send_max_retries,
$topic_request_required_acks =
$varnishkafka::defaults::topic_request_required_acks,
$topic_message_timeout_ms =
$varnishkafka::defaults::topic_message_timeout_ms,
+ $compression_codec =
$varnishkafka::defaults::compression_codec,
$varnish_opts = $varnishkafka::defaults::varnish_opts,
$log_data_copy = $varnishkafka::defaults::log_data_copy,
@@ -65,8 +69,6 @@
$log_level = $varnishkafka::defaults::log_level,
$log_stderr = $varnishkafka::defaults::log_stderr,
$log_syslog = $varnishkafka::defaults::log_syslog,
-
- $daemon_opts = $varnishkafka::defaults::daemon_opts,
$conf_template = $varnishkafka::defaults::conf_template,
$default_template = $varnishkafka::defaults::default_template
diff --git a/templates/varnishkafka.conf.erb b/templates/varnishkafka.conf.erb
index 54982be..6d1f32d 100644
--- a/templates/varnishkafka.conf.erb
+++ b/templates/varnishkafka.conf.erb
@@ -87,20 +87,20 @@
format.type = <%= @format_type %>
format = <%= @format %>
-# Optional secondary formatting.
-# 'output = kafka': The rendered 'format.key' will be provided as the
-# Kafka message Key
-# 'output = string': Print string to stdout.
-# Supports the same formating and type as 'format' and 'format.type'.
-#
<% if @format_key -%>
format.key.type = <%= @format_key_type %>
format.key = <%= @format_key %>
<% else -%>
# format.key.type = <%= @format_key_type %>
-# format.key = %h
+# format.key = %l
<% end -%>
+
+
+# The maximum accepted log tag size.
+# Larger tags will be truncated to this size.
+# Defaults to 2048
+#tag.size.max = 2048
# EXPERIMENTAL
@@ -112,8 +112,32 @@
# NOTE:
# Must be set to true for offline files (-r file..) due to the way
# libvarnishapi reads its data.
-log.data.copy = <%= @log_data_copy %>
+log.data.copy = true
+
+# TUNING
+# Logline cache hash tuning
+# 'log.hash.size * log.hash.max' dictates the maximum number of cached logline
+# entries in memory.
+
+# Number of hash buckets (keyed by log id).
+# Higher number yields more performance at the expense of memory.
+# Set this to avg_requests_per_second / 5.
+# Defaults to 5000
+#log.hash.size = 5000
+
+# Maximum number of loglines per hash bucket
+# Higher number yields less memory consumption at the expense of performance.
+# Set this to avg_requests_per_second / log.hash.size.
+# Defaults to 5
+#log.hash.max = 5
+
+# Size of per logline scratch buffer.
+# The scratch buffer is used as a temporary storage space while
+# collecting tags for the log line.
+# If the scratch size is too small the logline tag match will be incomplete.
+# Defaults to 4096 bytes.
+#log.line.scratch.size = 4096
# Start for sequence number (%n)
# Either a number, or the string "time" which will set it to the current
@@ -121,26 +145,40 @@
# Defaults to 0.
sequence.number = <%= @sequence_number %>
-# FUTURE: read last sequence number from file to allow restarts
-#sequence.file = /var/spool/varnishkafka.seq
-
-
#
# varnishkafka log messages configuration
# Debugging, error reporting, etc, not to be confused with varnish logs.
#
# varnishkafka log level (1 = emergencies .. 7 = debug)
-log.level = <%= @log_level %>
+log.level = 6
# specify log output (multiples allowed)
-log.stderr = <%= @log_stderr %>
-log.syslog = <%= @log_syslog %>>
+log.stderr = false
+log.syslog = true
+
+# Maximum number of error logs produced per log.rate.period seconds
+# This setting is applied per error type.
+# log.rate.max defaults to 100
+# log.rate.period defaults to 60
+#log.rate.max = 100
+#log.rate.period = 60
+
+# Kafka: log message delivery failures (requires required.acks > 0)
+log.kafka.msg.error = true
+
+# Statistics output interval
+# Statistics is collected from varnishkafka itself as well as librdkafka
+# and output as a log message of level LOG_INFO containing a JSON object
+# prefixed with either "STATS: " or "KAFKASTATS: ".
+# Defaults to 60 seconds, use 0 to disable.
+#log.statistics.interval = 60
# daemonize varnishkafka (boolean)
-daemonize = true
+daemonize = false
+
#######################################################################
@@ -154,7 +192,6 @@
# #
#######################################################################
-
<% @varnish_opts.each_pair do |option, value| -%>
# -<%= option %> <%= value %>
varnish.arg.<%= option %> = <%= value %>
@@ -167,7 +204,7 @@
# #
# For the full range of Kafka handle and topic configuration #
# properties, see: #
-# https://github.com/edenhill/librdkafka/blob/0.8-wip/rdkafka.h #
+# https://github.com/edenhill/librdkafka/blob/master/rdkafka.h #
# #
# And the Apache Kafka configuration reference: #
# http://kafka.apache.org/08/configuration.html #
@@ -178,10 +215,16 @@
metadata.broker.list = <%= Array(@brokers).join(',') %>
# Maximum number of messages allowed on the local producer queue
+# Defaults to 1000000
queue.buffering.max.messages = <%= @queue_buffering_max_messages %>
# Maximum number of retries per messageset.
message.send.max.retries = <%= @message_send_max_retries %>
+
+
+#
+# Topic configuration
+#
# Topic to produce messages to
topic = <%= @topic %>
@@ -195,4 +238,7 @@
# Local message timeout (milliseconds)
topic.message.timeout.ms = <%= topic_message_timeout_ms %>
+# Use compression when sending to Kafka.. Default is none.
+# Valid values are 'none', 'gzip', and 'snappy'.
+compression.codec = <%= @compression_codec %>
<% end -%>
--
To view, visit https://gerrit.wikimedia.org/r/91664
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Id856d1f9a2cbc1bb6f22dc7bd95c2df950a10976
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet/varnishkafka
Gerrit-Branch: master
Gerrit-Owner: Ottomata <o...@wikimedia.org>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
