Ottomata has uploaded a new change for review. https://gerrit.wikimedia.org/r/82885
Change subject: varnishkafka module, first real commit! ...................................................................... varnishkafka module, first real commit! Change-Id: If2a6dd7c7ac353d65792ee1559a0d2db351feb44 --- M README.md A manifests/defaults.pp A manifests/init.pp A templates/varnishkafka.conf.erb A templates/varnishkafka.default.erb 5 files changed, 355 insertions(+), 0 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/operations/puppet/varnishkafka refs/changes/85/82885/1 diff --git a/README.md b/README.md index e69de29..77debbc 100644 --- a/README.md +++ b/README.md @@ -0,0 +1,13 @@ +# varnishkafka puppet module + +Puppet module for [varnishkafka](https://github.com/wikimedia/varnishkafka). + +# Usage: + +```puppet +class { 'varnishkafka': + brokers => ['kafka1.domain.org:9092', 'broker2.domain.org:9092'], + format_type => 'json', + format => '%{@hostname}l %{@sequence!num}n %{%FT%T@dt}t %{Varnish:time_firstbyte@time_firstbyte!num}x %{@ip}h %{Varnish:handling@cache_status}x %{@http_status}s %{@response_size!num}b %{@http_method}m %{Host@uri_host}i %{@uri_path}U %{@uri_query}q %{Content-Type@content_type}o %{Referer@referer}i %{X-Forwarded-For@x_forwarded_for}i %{User-Agent@user_agent}i %{Accept-Language@accept_language}i' +} +``` diff --git a/manifests/defaults.pp b/manifests/defaults.pp new file mode 100644 index 0000000..36e975b --- /dev/null +++ b/manifests/defaults.pp @@ -0,0 +1,34 @@ +# == Class varnishkafka::defaults +# +class varnishkafka::defaults { + $brokers = ['localhost:9092'] + $topic = 'varnish' + $sequence_number = 0 + + $output = 'kafka' + $format_type = 'string' + $format = '%l %n %t %{Varnish:time_firstbyte}x %h %{Varnish:handling}x/%s %b %m http://%{Host}i%U%q - %{Content-Type}o %{Referer}i %{X-Forwarded-For}i %{User-agent!escape}i' + + $format_key_type = 'kafka' + $format_key = undef + + $partition = -1 + $queue_buffering_max_messages = 1000000 + $message_send_max_retries = 3 + $topic_request_required_acks = 1 + $topic_message_timeout_ms = 60000 + + $varnish_opts = { + 'm' => 'RxRequest:^(?!PURGE$)', + } + $log_data_copy = true + + $log_level = 6 + $log_stderr = true + $log_syslog = true + + $daemon_opts = undef + + $conf_template = 'varnishkafka/varnishkafka.conf.erb' + $default_template = 'varnishkafka/varnishkafka.default.erb' +} diff --git a/manifests/init.pp b/manifests/init.pp new file mode 100644 index 0000000..9c0bf8e --- /dev/null +++ b/manifests/init.pp @@ -0,0 +1,97 @@ +# == Class varnishkafka +# Configures and runs varnishkafka Varnish to Kafka producer. +# See: https://github.com/wikimedia/varnishkafka +# Most varnishkafka.conf properties are supported. +# +# == Parameters +# $brokers - Array of Kafka broker host:ports. +# Default: [localhost:9091] +# $topic - Kafka topic name to produce to. +# Default: varnish +# $sequence_number - Sequence number at which to start logging. +# You can set this to an arbitrary integer, or to +# 'time', which will start the sequence number +# at the current timestamp * 10000000. Default: 0 +# $output - output type. Either 'kafka', 'stdout', or 'null'. +# Default: kafka +# $format_type - Log format type. Either 'string' or 'json' +# Default: string +# $format - Log format string. +# $format_key_type - Kafka message key format type. +# Either 'string' or 'json'. Default: string +# $format_key - Kafka message ey format string. +# Default: undef (disables Kafka message key usage). +# $partition - Topic partition number to send to. -1 for random. +# Default: -1. +# $queue_buffering_max_messages - Maximum number of messages allowed on the +# local Kafka producer queue. Default: 1000000 +# $message_send_max_retries - Maximum number of retries per messageset. +# Default: 3 +# $topic_request_required_acks - Required ack level. Default: 1 +# $topic_message_timeout_ms - Local message timeout (milliseconds). +# Default: 60000 +# $varnish_opts - Arbitrary hash of varnish CLI options. +# Default: { 'm' => 'RxRequest:^(?!PURGE$)' } +# $log_data_copy - If true, log tag data read from VSL files +# should be copied instantly when read. Default true. +# $log_level - varnishkafka log level. Default 6 (info). +# $log_stderr - Boolean. Whether to log to stderr. Default: true +# $log_syslog - Boolean. Whether to log to syslog. Default: true +# $daemon_opts - /etc/default/kafka DAEMON_OPTS to pass to start-stop-daemon. +# +# $conf_template +# $default_template +# +class varnishkafka( + $brokers = $varnishkafka::defaults::brokers, + $topic = $varnishkafka::defaults::topic, + $sequence_number = $varnishkafka::defaults::sequence_number, + + $output = $varnishkafka::defaults::output, + $format_type = $varnishkafka::defaults::format_type, + $format = $varnishkafka::defaults::format, + + $format_key_type = $varnishkafka::defaults::format_key_type, + $format_key = $varnishkafka::defaults::format_key, + + $partition = $varnishkafka::defaults::partition, + $queue_buffering_max_messages = $varnishkafka::defaults::queue_buffering_max_messages, + $message_send_max_retries = $varnishkafka::defaults::message_send_max_retries, + $topic_request_required_acks = $varnishkafka::defaults::topic_request_required_acks, + $topic_message_timeout_ms = $varnishkafka::defaults::topic_message_timeout_ms, + + $varnish_opts = $varnishkafka::defaults::varnish_opts, + $log_data_copy = $varnishkafka::defaults::log_data_copy, + + $log_level = $varnishkafka::defaults::log_level, + $log_stderr = $varnishkafka::defaults::log_stderr, + $log_syslog = $varnishkafka::defaults::log_syslog, + + $daemon_opts = $varnishkafka::defaults::daemon_opts, + + $conf_template = $varnishkafka::defaults::conf_template, + $default_template = $varnishkafka::defaults::default_template +) inherits varnishkafka::defaults +{ + package { 'varnishkafka': + ensure => 'installed', + } + + file { '/etc/varnishkafka.conf': + content => template($conf_template), + require => Package['varnishkafka'], + } + + file { '/etc/default/varnishkafka': + content => template($default_template), + require => Package['varnishkafka'], + } + + service { 'varnishkafka': + ensure => 'running', + enable => true, + hasstatus => true, + hasrestart => true, + subscribe => [File['/etc/varnishkafka.conf'], File['/etc/default/varnishkafka']], + } +} \ No newline at end of file diff --git a/templates/varnishkafka.conf.erb b/templates/varnishkafka.conf.erb new file mode 100644 index 0000000..30971fb --- /dev/null +++ b/templates/varnishkafka.conf.erb @@ -0,0 +1,198 @@ +# Note: This file is managed by Puppet. + +####################################################################### +# # +# varnishkafka configuration file # +# # +# # +####################################################################### +# # +# Syntax: # +# <property-name> = <value> # +# # +# Boolean property values: # +# >0, "true", "yes", "on" - interpreted as true # +# everything else - interpreted as false # +# # +####################################################################### + # + # + # +####################################################################### +# # +# Varnish log formatting # +# # +# format.type - format output type, one of: # +# string - ASCII string output # +# json - JSON output # +# # +# # +# format - format string # +# %X # +# where 'X' is one of the standard varnishncsa(1) formatters. # +# Example: %u # +# # +# # +# %{VAR}X # +# Name-Value tokens where X is 'x', 'i' or 'o' and 'VAR' is the # +# Name to extract the value for. # +# Example: %{User-Agent}i # +# # +# # +# %{?DEFAULT@FIELD!OPTION!OPTION..}X # +# where 'X' is any formatter, # +# # +# 'DEFAULT' is the default string to use if no tag was matched, # +# the default default string is "-". # +# # +# 'FIELD' is the field name to use with the JSON formatter. # +# i.e., "%{@host}l" will be JSON encoded as: {"host":"1.2.3.4"} # +# # +# 'OPTION' is one or more of the formatting options: # +# escape - escape non-printable characters to \<octalcode> # +# and \t\n\r\v\f " to their canonical # +# backslashed notations (\t\n\r\v\f\"\ ). # +# num - for typed formatters, such as JSON, try to encode # +# the value as a number. # +# # +# # +# This syntax can be combined with %{VAR}X. # +# Example: %{User-Agent?Mozilla!escape}i # +# %{?nouser}u # +# %{!escape}q # +# %{@host}l # +# # +# # +# # +# Additional formatter specials: # +# %{<strftime-format>}t - format timestamp according to supplied # +# strftime(3) compatible format string. # +# # +# # +# # +# Non %-prefixed strings are copied verbatim to the # +# output log string. # +# Example: "User: %u;" would render "User: snaps;" # +# # +# # +####################################################################### + +# Where to output varnish log lines: +# kafka - (default) send to kafka broker +# stdout - just print to stdout (behave like varnishncsa) +# null - (test) collect all tags specified by format but dont output anything +output = <%= @output %> + +# Log formatter +format.type = <%= @format_type %> +format = <%= @format %> + +# Optional secondary formatting. +# 'output = kafka': The rendered 'format.key' will be provided as the +# Kafka message Key +# 'output = string': Print string to stdout. +# Supports the same formating and type as 'format' and 'format.type'. +# +<% if @format_key -%> +format.key.type = <%= @format_key_type %> +format.key = <%= @format_key %> +<% else -%> +# format.key.type = <%= @format_key_type %> +# format.key = %{%s}h +<% end -%> + + + +# EXPERIMENTAL +# Indicates if the log tag data read from VSL files should be copied instantly +# when read (true). If this is set to false the data is assumed to be +# persistent (for the duration of collecting and formatting a single request) +# and no copies will be made, thus improving performance. +# +# NOTE: +# Must be set to true for offline files (-r file..) due to the way +# libvarnishapi reads its data. +log.data.copy = <%= @log_data_copy %> + + +# Start for sequence number (%n) +# Either a number, or the string "time" which will set it to the current +# unix time in seconds multiplied by 1,000,000. +# Defaults to 0. +sequence.number = <%= @sequence_number %> + +# FUTURE: read last sequence number from file to allow restarts +#sequence.file = /var/spool/varnishkafka.seq + + +# +# varnishkafka log messages configuration +# Debugging, error reporting, etc, not to be confused with varnish logs. +# + +# varnishkafka log level (1 = emergencies .. 7 = debug) +log.level = <%= @log_level %> + +# specify log output (multiples allowed) +log.stderr = <%= @log_stderr %> +log.syslog = <%= @log_syslog %>> + + + +# daemonize varnishkafka (boolean) +daemonize = true + + +####################################################################### +# # +# Standard varnish VSL command line arguments # +# # +# Syntax: # +# varnish.arg.<c> = <value>, where <c> is a command line option. # +# # +# See varnishncsa(1) and varnishlog(1) for valid options. # +# # +####################################################################### + + +<% @varnish_opts.each_pair do |option, value| -%> +# -<%= option %> <%= value %> +varnish.arg.<%= option %> = <%= value %> +<% end -%> + +<% if @output == 'kafka' -%> +####################################################################### +# # +# Kafka configuration # +# # +# For the full range of Kafka handle and topic configuration # +# properties, see: # +# https://github.com/edenhill/librdkafka/blob/0.8-wip/rdkafka.h # +# # +# And the Apache Kafka configuration reference: # +# http://kafka.apache.org/08/configuration.html # +# # +####################################################################### + +# Initial list of kafka brokers +metadata.broker.list = <%= Array(@brokers).join(',') %> + +# Maximum number of messages allowed on the local producer queue +queue.buffering.max.messages = <%= @queue_buffering_max_messages %> + +# Maximum number of retries per messageset. +message.send.max.retries = <%= @message_send_max_retries %> + +# Topic to produce messages to +topic = <%= @topic %> + +# Partition (-1: random, else one of the available partitions) +partition = <%= @partition %> + +# Required number of acks +topic.request.required.acks = <%= @topic_request_required_acks %> + +# Local message timeout (milliseconds) +topic.message.timeout.ms = <%= topic_message_timeout_ms %> + +<% end -%> diff --git a/templates/varnishkafka.default.erb b/templates/varnishkafka.default.erb new file mode 100644 index 0000000..8fd11be --- /dev/null +++ b/templates/varnishkafka.default.erb @@ -0,0 +1,13 @@ +# Note: This file is managed by Puppet. + +# Configuration file for varnishkafka +# +# This shell script fragment is sourced by /etc/init.d/varnishkafka +# +VARNISHKAFKA_ENABLED=1 + +<% if @daemon_opts -%> +DAEMON_OPTS=<%= @daemon_opts -%> +<% else -%> +# DAEMON_OPTS="-a -w ${LOGFILE} -D -P ${PIDFILE}" +<% end -%> -- To view, visit https://gerrit.wikimedia.org/r/82885 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: If2a6dd7c7ac353d65792ee1559a0d2db351feb44 Gerrit-PatchSet: 1 Gerrit-Project: operations/puppet/varnishkafka Gerrit-Branch: master Gerrit-Owner: Ottomata <[email protected]> _______________________________________________ MediaWiki-commits mailing list [email protected] https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
