Ottomata has submitted this change and it was merged.
Change subject: Updating with recent upstream changes to varnishkafka.conf
......................................................................
Updating with recent upstream changes to varnishkafka.conf
Change-Id: Id856d1f9a2cbc1bb6f22dc7bd95c2df950a10976
---
M manifests/defaults.pp
M manifests/init.pp
M templates/varnishkafka.conf.erb
3 files changed, 111 insertions(+), 32 deletions(-)
Approvals:
Ottomata: Verified; Looks good to me, approved
diff --git a/manifests/defaults.pp b/manifests/defaults.pp
index 36e975b..756c444 100644
--- a/manifests/defaults.pp
+++ b/manifests/defaults.pp
@@ -7,9 +7,9 @@
$output = 'kafka'
$format_type = 'string'
- $format = '%l %n %t
%{Varnish:time_firstbyte}x %h %{Varnish:handling}x/%s %b %m
http://%{Host}i%U%q - %{Content-Type}o %{Referer}i
%{X-Forwarded-For}i %{User-agent!escape}i'
+ $format = '%l %n %t
%{Varnish:time_firstbyte}x %h %{Varnish:handling}x/%s %b %m
http://%{Host}i%U%q - %{Content-Type}o %{Referer}i
%{X-Forwarded-For}i %{User-agent!escape}i %{Accept-Language}i'
- $format_key_type = 'kafka'
+ $format_key_type = 'string'
$format_key = undef
$partition = -1
@@ -17,17 +17,20 @@
$message_send_max_retries = 3
$topic_request_required_acks = 1
$topic_message_timeout_ms = 60000
+ $compression_codec = 'none'
$varnish_opts = {
'm' => 'RxRequest:^(?!PURGE$)',
}
$log_data_copy = true
+ $tag_size_max = 2048
+ $log_line_scratch_size = 4096
+ $log_hash_size = 5000
+ $log_hash_max = 5
$log_level = 6
- $log_stderr = true
+ $log_stderr = false
$log_syslog = true
-
- $daemon_opts = undef
$conf_template = 'varnishkafka/varnishkafka.conf.erb'
$default_template = 'varnishkafka/varnishkafka.default.erb'
diff --git a/manifests/init.pp b/manifests/init.pp
index 3a39eae..1b9898f 100644
--- a/manifests/init.pp
+++ b/manifests/init.pp
@@ -30,10 +30,24 @@
# $topic_request_required_acks - Required ack level. Default: 1
# $topic_message_timeout_ms - Local message timeout (milliseconds).
# Default: 60000
+# $compression_codec - Compression codec to use when sending
batched messages
+# Kafka. Valid values are 'none', 'gzip',
and 'snappy'.
+# Default: none
# $varnish_opts - Arbitrary hash of varnish CLI options.
# Default: { 'm' =>
'RxRequest:^(?!PURGE$)' }
# $log_data_copy - If true, log tag data read from VSL files
# should be copied instantly when read.
Default true.
+# $tag_size_max - Maximum size of an individual field.
Field will be truncated
+# if it is larger than this. Default: 2048
+# $log_line_scratch_size - Size of static log line buffer. If a
line is larger than
+# this buffer, temp buffers will be
allocated. Set this
+# slighly larger than your expected line
size.
+# Default: 4096
+# $log_hash_size - Number of hash buckets. Set this to
avg_requests_per_second / 5.
+# Default: 5000
+# $log_hash_max - Max number of log lines / bucket. Set
this to
+# avg_requests_per_second / $log_hash_size.
+# Default: 5
# $log_level - varnishkafka log level. Default 6
(info).
# $log_stderr - Boolean. Whether to log to stderr.
Default: true
# $log_syslog - Boolean. Whether to log to syslog.
Default: true
@@ -58,15 +72,18 @@
$message_send_max_retries =
$varnishkafka::defaults::message_send_max_retries,
$topic_request_required_acks =
$varnishkafka::defaults::topic_request_required_acks,
$topic_message_timeout_ms =
$varnishkafka::defaults::topic_message_timeout_ms,
+ $compression_codec =
$varnishkafka::defaults::compression_codec,
$varnish_opts = $varnishkafka::defaults::varnish_opts,
$log_data_copy = $varnishkafka::defaults::log_data_copy,
+ $tag_size_max = $varnishkafka::defaults::tag_size_max,
+ $log_line_scratch_size =
$varnishkafka::defaults::log_line_scratch_size,
+ $log_hash_size = $varnishkafka::defaults::log_hash_size,
+ $log_hash_max = $varnishkafka::defaults::log_hash_max,
$log_level = $varnishkafka::defaults::log_level,
$log_stderr = $varnishkafka::defaults::log_stderr,
$log_syslog = $varnishkafka::defaults::log_syslog,
-
- $daemon_opts = $varnishkafka::defaults::daemon_opts,
$conf_template = $varnishkafka::defaults::conf_template,
$default_template = $varnishkafka::defaults::default_template
diff --git a/templates/varnishkafka.conf.erb b/templates/varnishkafka.conf.erb
index 54982be..4daaa34 100644
--- a/templates/varnishkafka.conf.erb
+++ b/templates/varnishkafka.conf.erb
@@ -91,29 +91,14 @@
# 'output = kafka': The rendered 'format.key' will be provided as the
# Kafka message Key
# 'output = string': Print string to stdout.
-# Supports the same formating and type as 'format' and 'format.type'.
-#
+# Supports the same formatting and type as 'format' and 'format.type'.
<% if @format_key -%>
format.key.type = <%= @format_key_type %>
format.key = <%= @format_key %>
<% else -%>
# format.key.type = <%= @format_key_type %>
-# format.key = %h
+# format.key = %l
<% end -%>
-
-
-
-# EXPERIMENTAL
-# Indicates if the log tag data read from VSL files should be copied instantly
-# when read (true). If this is set to false the data is assumed to be
-# persistent (for the duration of collecting and formatting a single request)
-# and no copies will be made, thus improving performance.
-#
-# NOTE:
-# Must be set to true for offline files (-r file..) due to the way
-# libvarnishapi reads its data.
-log.data.copy = <%= @log_data_copy %>
-
# Start for sequence number (%n)
# Either a number, or the string "time" which will set it to the current
@@ -121,8 +106,56 @@
# Defaults to 0.
sequence.number = <%= @sequence_number %>
-# FUTURE: read last sequence number from file to allow restarts
-#sequence.file = /var/spool/varnishkafka.seq
+
+#
+# TUNING
+#
+
+# The maximum accepted log tag (field) size.
+# Larger tags will be truncated to this size.
+# Defaults to 2048
+tag.size.max = <%= @tag_size_max %>
+
+# Size of per logline scratch buffer.
+# The scratch buffer is used as a temporary storage space while
+# collecting tags for the log line. If the scratch size is too small the
+# logline tag match will be incomplete. log.line.scratch.size depicts the
+# size of the "static" always-available memory allocated with each logline.
+# It should be configured to fit all matched tag-values (prior to formatting)
+# for a normal request. If the scratch buffer gets full vk will start
+# allocating tmpbufs, a tmpbuf only lives for the current request and is then
+# freed, so it is a little more costly than using thestatic scratch pad.
+# Defaults to 4096 bytes.
+log.line.scratch.size = <%= @log_line_scratch_size %>
+
+
+# Logline cache hash tuning
+# 'log.hash.size * log.hash.max' dictates the maximum number of cached logline
+# entries in memory.
+
+# Number of hash buckets (keyed by log id).
+# Higher number yields more performance at the expense of memory.
+# Set this to avg_requests_per_second / 5.
+# Defaults to 5000
+log.hash.size = <%= @log_hash_size %>
+
+# Maximum number of loglines per hash bucket
+# Higher number yields less memory consumption at the expense of performance.
+# Set this to avg_requests_per_second / log.hash.size.
+# Defaults to 5
+log.hash.max = <%= @log_hash_max %>
+
+
+# EXPERIMENTAL
+# log.data.copy indicates if the log tag data read from VSL files should be
copied
+# instantly when read (true). If this is set to false the data is assumed to be
+# persistent (for the duration of collecting and formatting a single request)
+# and no copies will be made, thus improving performance.
+#
+# NOTE:
+# Must be set to true for offline files (-r file..) due to the way
+# libvarnishapi reads its data.
+log.data.copy = true
#
@@ -131,16 +164,34 @@
#
# varnishkafka log level (1 = emergencies .. 7 = debug)
-log.level = <%= @log_level %>
+log.level = 6
# specify log output (multiples allowed)
-log.stderr = <%= @log_stderr %>
-log.syslog = <%= @log_syslog %>>
+log.stderr = false
+log.syslog = true
+
+# Maximum number of error logs produced per log.rate.period seconds
+# This setting is applied per error type.
+# log.rate.max defaults to 100
+# log.rate.period defaults to 60
+#log.rate.max = 100
+#log.rate.period = 60
+
+# Kafka: log message delivery failures (requires required.acks > 0)
+log.kafka.msg.error = true
+
+# Statistics output interval
+# Statistics is collected from varnishkafka itself as well as librdkafka
+# and output as a log message of level LOG_INFO containing a JSON object
+# prefixed with either "STATS: " or "KAFKASTATS: ".
+# Defaults to 60 seconds, use 0 to disable.
+#log.statistics.interval = 60
# daemonize varnishkafka (boolean)
-daemonize = true
+daemonize = false
+
#######################################################################
@@ -154,7 +205,6 @@
# #
#######################################################################
-
<% @varnish_opts.each_pair do |option, value| -%>
# -<%= option %> <%= value %>
varnish.arg.<%= option %> = <%= value %>
@@ -167,7 +217,7 @@
# #
# For the full range of Kafka handle and topic configuration #
# properties, see: #
-# https://github.com/edenhill/librdkafka/blob/0.8-wip/rdkafka.h #
+# https://github.com/edenhill/librdkafka/blob/master/rdkafka.h #
# #
# And the Apache Kafka configuration reference: #
# http://kafka.apache.org/08/configuration.html #
@@ -178,10 +228,16 @@
metadata.broker.list = <%= Array(@brokers).join(',') %>
# Maximum number of messages allowed on the local producer queue
+# Defaults to 1000000
queue.buffering.max.messages = <%= @queue_buffering_max_messages %>
# Maximum number of retries per messageset.
message.send.max.retries = <%= @message_send_max_retries %>
+
+
+#
+# Topic configuration
+#
# Topic to produce messages to
topic = <%= @topic %>
@@ -195,4 +251,7 @@
# Local message timeout (milliseconds)
topic.message.timeout.ms = <%= topic_message_timeout_ms %>
+# Use compression when sending to Kafka.. Default is none.
+# Valid values are 'none', 'gzip', and 'snappy'.
+compression.codec = <%= @compression_codec %>
<% end -%>
--
To view, visit https://gerrit.wikimedia.org/r/91664
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Id856d1f9a2cbc1bb6f22dc7bd95c2df950a10976
Gerrit-PatchSet: 3
Gerrit-Project: operations/puppet/varnishkafka
Gerrit-Branch: master
Gerrit-Owner: Ottomata <o...@wikimedia.org>
Gerrit-Reviewer: Edenhill <mag...@edenhill.se>
Gerrit-Reviewer: Ottomata <o...@wikimedia.org>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
