Ottomata has submitted this change and it was merged.
Change subject: Allow multiple instances
......................................................................
Allow multiple instances
Allow multiple varnishkafka instances per node by changing the ::varnishkafka
class to a custom resource type -- ::varnishkafa::instance.
Each instance is managed via an upstart config
/etc/init/varnishkafka-${name}.conf
and a config file /etc/varnishkafka/${name}.conf.
Tested.
Change-Id: I205f92c72e3da23771ce2223b6eb8ee48ac95c5b
---
D manifests/defaults.pp
M manifests/init.pp
A manifests/instance.pp
A manifests/monitor.pp
D manifests/monitoring.pp
D templates/varnishkafka.default.erb
A templates/varnishkafka.upstart.conf.erb
7 files changed, 230 insertions(+), 226 deletions(-)
Approvals:
Ottomata: Verified; Looks good to me, approved
diff --git a/manifests/defaults.pp b/manifests/defaults.pp
deleted file mode 100644
index a8f76a9..0000000
--- a/manifests/defaults.pp
+++ /dev/null
@@ -1,46 +0,0 @@
-# == Class varnishkafka::defaults
-#
-class varnishkafka::defaults {
- $brokers = ['localhost:9092']
- $topic = 'varnish'
- $sequence_number = 0
-
- $output = 'kafka'
- $format_type = 'string'
- $format = '%l %n %t
%{Varnish:time_firstbyte}x %h %{Varnish:handling}x/%s %b %m
http://%{Host}i%U%q - %{Content-Type}o %{Referer}i
%{X-Forwarded-For}i %{User-agent!escape}i %{Accept-Language}i'
-
- $format_key_type = 'string'
- $format_key = undef
-
- $partition = -1
- $queue_buffering_max_messages = 100000
- $queue_buffering_max_ms = 1000
- $batch_num_messages = 1000
- $message_send_max_retries = 3
- $topic_request_required_acks = 1
- $topic_message_timeout_ms = 300000
- $topic_request_timeout_ms = 5000
- $socket_send_buffer_bytes = 0
- $compression_codec = 'none'
-
- $varnish_name = undef
- $varnish_opts = {
- 'm' => 'RxRequest:^(?!PURGE$)',
- }
-
- $tag_size_max = 2048
- $logline_scratch_size = 4096
- $logline_hash_size = 5000
- $logline_hash_max = 5
- $logline_data_copy = true
-
- $log_level = 6
- $log_stderr = false
- $log_syslog = true
- $log_statistics_file =
'/var/cache/varnishkafka/varnishkafka.stats.json'
- $log_statistics_interval = 60
-
- $should_subscribe = true
- $conf_template = 'varnishkafka/varnishkafka.conf.erb'
- $default_template = 'varnishkafka/varnishkafka.default.erb'
-}
diff --git a/manifests/init.pp b/manifests/init.pp
index 21f3593..19fe8a6 100644
--- a/manifests/init.pp
+++ b/manifests/init.pp
@@ -1,149 +1,52 @@
# == Class varnishkafka
# Configures and runs varnishkafka Varnish to Kafka producer.
# See: https://github.com/wikimedia/varnishkafka
-# Most varnishkafka.conf properties are supported.
#
-# == Parameters
-# $brokers - Array of Kafka broker host:ports.
-# Default: [localhost:9091]
-# $topic - Kafka topic name to produce to.
-# Default: varnish
-# $sequence_number - Sequence number at which to start
logging.
-# You can set this to an arbitrary
integer, or to
-# 'time', which will start the sequence
number
-# at the current timestamp * 10000000.
Default: 0
-# $output - output type. Either 'kafka', 'stdout',
or 'null'.
-# Default: kafka
-# $format_type - Log format type. Either 'string' or
'json'
-# Default: string
-# $format - Log format string.
-# $format_key_type - Kafka message key format type.
-# Either 'string' or 'json'. Default:
string
-# $format_key - Kafka message key format string.
-# Default: undef (disables Kafka message
key usage).
-# $partition - Topic partition number to send to. -1
for random.
-# Default: -1
-# $queue_buffering_max_messages - Maximum number of messages allowed on the
-# local Kafka producer queue. Default:
100000
-# $queue_buffering_max_ms - Maximum time, in milliseconds, for
buffering
-# data on the producer queue. Default:
1000
-# $batch_num_messages - Maximum number of messages batched in
one MessageSet.
-# Default: 1000
-# $message_send_max_retries - Maximum number of retries per messageset.
-# Default: 3
-# $topic_request_required_acks - Required ack level. Default: 1
-# $topic_message_timeout_ms - Local message timeout (milliseconds).
-# Default: 300000
-# $topic_request_timeout_ms - Ack timeout of the produce request.
-# Default: 5000
-# $socket_send_buffer_bytes - SO_SNDBUFF Socket send buffer size.
System default is used if 0.
-# Default: 0
-# $compression_codec - Compression codec to use when sending
batched messages to
-# Kafka. Valid values are 'none', 'gzip',
and 'snappy'.
-# Default: none
-# $varnish_name - Name of varnish instance to log from.
Default: undef
-# $varnish_opts - Arbitrary hash of varnish CLI options.
-# Default: { 'm' =>
'RxRequest:^(?!PURGE$)' }
-# $tag_size_max - Maximum size of an individual field.
Field will be truncated
-# if it is larger than this. Default: 2048
-# $logline_line_scratch_size - Size of static log line buffer. If a
line is larger than
-# this buffer, temp buffers will be
allocated. Set this
-# slighly larger than your expected line
size.
-# Default: 4096
-# $logline_hash_size - Number of hash buckets. Set this to
avg_requests_per_second / 5.
-# Default: 5000
-# $logline_hash_max - Max number of log lines / bucket. Set
this to
-# avg_requests_per_second / $log_hash_size.
-# Default: 5
-# $logline_data_copy - If true, log tag data read from VSL files
-# should be copied instantly when read.
Default true.
-# $log_level - varnishkafka log level. Default 6
(info).
-# $log_stderr - Boolean. Whether to log to stderr.
Default: true
-# $log_syslog - Boolean. Whether to log to syslog.
Default: true
-# $log_statistics_file - Path to varnishkafka JSON statistics
file.
-# Default:
/var/cache/varnishkafka/varnishkafka.stats.json
-# $log_statistics_interval - JSON statistics file output interval in
seconds. Default: 60
-#
-# $should_subscribe - If true, the varnishkafka service will
restart for config
-# changes. Default: true.
-# $conf_template
-# $default_template
-#
-class varnishkafka(
- $brokers = $varnishkafka::defaults::brokers,
- $topic = $varnishkafka::defaults::topic,
- $sequence_number = $varnishkafka::defaults::sequence_number,
-
- $output = $varnishkafka::defaults::output,
- $format_type = $varnishkafka::defaults::format_type,
- $format = $varnishkafka::defaults::format,
-
- $format_key_type = $varnishkafka::defaults::format_key_type,
- $format_key = $varnishkafka::defaults::format_key,
-
- $partition = $varnishkafka::defaults::partition,
- $queue_buffering_max_messages =
$varnishkafka::defaults::queue_buffering_max_messages,
- $queue_buffering_max_ms =
$varnishkafka::defaults::queue_buffering_max_ms,
- $batch_num_messages =
$varnishkafka::defaults::batch_num_messages,
-
- $message_send_max_retries =
$varnishkafka::defaults::message_send_max_retries,
- $topic_request_required_acks =
$varnishkafka::defaults::topic_request_required_acks,
- $topic_message_timeout_ms =
$varnishkafka::defaults::topic_message_timeout_ms,
- $topic_request_timeout_ms =
$varnishkafka::defaults::topic_request_timeout_ms,
- $socket_send_buffer_bytes =
$varnishkafka::defaults::socket_send_buffer_bytes,
- $compression_codec =
$varnishkafka::defaults::compression_codec,
-
- $varnish_opts = $varnishkafka::defaults::varnish_opts,
- $varnish_name = $varnishkafka::defaults::varnish_name,
-
- $tag_size_max = $varnishkafka::defaults::tag_size_max,
- $logline_scratch_size =
$varnishkafka::defaults::logline_scratch_size,
- $logline_hash_size =
$varnishkafka::defaults::logline_hash_size,
- $logline_hash_max =
$varnishkafka::defaults::logline_hash_max,
- $logline_data_copy =
$varnishkafka::defaults::logline_data_copy,
-
- $log_level = $varnishkafka::defaults::log_level,
- $log_stderr = $varnishkafka::defaults::log_stderr,
- $log_syslog = $varnishkafka::defaults::log_syslog,
- $log_statistics_file =
$varnishkafka::defaults::log_statistics_file,
- $log_statistics_interval =
$varnishkafka::defaults::log_statistics_interval,
-
- $should_subscribe =
$varnishkafka::defaults::should_subscribe,
- $conf_template = $varnishkafka::defaults::conf_template,
- $default_template = $varnishkafka::defaults::default_template
-) inherits varnishkafka::defaults
-{
+class varnishkafka {
package { 'varnishkafka':
- ensure => 'present',
+ ensure => present,
}
- file { '/etc/varnishkafka.conf':
- content => template($conf_template),
+ file { '/etc/varnishkafka':
+ ensure => directory,
+ owner => 'root',
+ group => 'root',
+ mode => '0444',
+ recurse => true,
+ purge => true,
+ force => true,
+ }
+
+ # Don't use the init script provided by the package, because it precludes
+ # running multiple instances.
+
+ file { '/var/cache/varnishkafka':
+ ensure => directory,
+ owner => 'varnishlog',
+ group => 'varnishlog',
+ mode => '0755',
require => Package['varnishkafka'],
+ }
+
+ # Managing the varnishkafka service via its init script requires that the
+ # init script be present and that the default file mark the service as
+ # enabled. Invoking start-stop-daemon directly allows us to manage the
+ # service without having a tricky ordering dependency on those two
+ # resources.
+
+ exec { 'stop-varnishkafka-service':
+ command => '/sbin/start-stop-daemon --stop --pidfile
/var/run/varnishkafka/varnishkafka.pid --exec /usr/bin/varnishkafka',
+ onlyif => '/sbin/start-stop-daemon --status --pidfile
/var/run/varnishkafka/varnishkafka.pid --exec /usr/bin/varnishkafka',
+ subscribe => Package['varnishkafka'],
+ }
+
+ file { '/etc/init.d/varnishkafka':
+ ensure => absent,
+ subscribe => Package['varnishkafka'],
}
file { '/etc/default/varnishkafka':
- content => template($default_template),
- require => Package['varnishkafka'],
+ ensure => absent,
+ subscribe => Package['varnishkafka'],
}
-
- service { 'varnishkafka':
- ensure => 'running',
- enable => true,
- hasstatus => true,
- hasrestart => true,
- }
-
- # subscribe varnishkafka to its config files
- if $should_subscribe {
- Service['varnishkafka'] {
- subscribe => [File['/etc/varnishkafka.conf'],
File['/etc/default/varnishkafka']],
- }
- }
- # else just require them
- else {
- Service['varnishkafka'] {
- require => [File['/etc/varnishkafka.conf'],
File['/etc/default/varnishkafka']],
- }
- }
-}
\ No newline at end of file
+}
diff --git a/manifests/instance.pp b/manifests/instance.pp
new file mode 100644
index 0000000..fb0d67a
--- /dev/null
+++ b/manifests/instance.pp
@@ -0,0 +1,137 @@
+# == Class varnishkafka
+# Configures and runs varnishkafka Varnish to Kafka producer.
+# See: https://github.com/wikimedia/varnishkafka
+# Most varnishkafka.conf properties are supported.
+#
+# == Parameters
+# $brokers - Array of Kafka broker host:ports.
+# Default: [localhost:9091]
+# $topic - Kafka topic name to produce to.
+# Default: varnish
+# $sequence_number - Sequence number at which to start
logging.
+# You can set this to an arbitrary
integer, or to
+# 'time', which will start the sequence
number
+# at the current timestamp * 10000000.
Default: 0
+# $output - output type. Either 'kafka', 'stdout',
or 'null'.
+# Default: kafka
+# $format_type - Log format type. Either 'string' or
'json'
+# Default: string
+# $format - Log format string.
+# $format_key_type - Kafka message key format type.
+# Either 'string' or 'json'. Default:
string
+# $format_key - Kafka message key format string.
+# Default: undef (disables Kafka message
key usage).
+# $partition - Topic partition number to send to. -1
for random.
+# Default: -1
+# $queue_buffering_max_messages - Maximum number of messages allowed on the
+# local Kafka producer queue. Default:
100000
+# $queue_buffering_max_ms - Maximum time, in milliseconds, for
buffering
+# data on the producer queue. Default:
1000
+# $batch_num_messages - Maximum number of messages batched in
one MessageSet.
+# Default: 1000
+# $message_send_max_retries - Maximum number of retries per messageset.
+# Default: 3
+# $topic_request_required_acks - Required ack level. Default: 1
+# $topic_message_timeout_ms - Local message timeout (milliseconds).
+# Default: 300000
+# $topic_request_timeout_ms - Ack timeout of the produce request.
+# Default: 5000
+# $socket_send_buffer_bytes - SO_SNDBUFF Socket send buffer size.
System default is used if 0.
+# Default: 0
+# $compression_codec - Compression codec to use when sending
batched messages to
+# Kafka. Valid values are 'none', 'gzip',
and 'snappy'.
+# Default: none
+# $varnish_name - Name of varnish instance to log from.
Default: undef
+# $varnish_opts - Arbitrary hash of varnish CLI options.
+# Default: { 'm' =>
'RxRequest:^(?!PURGE$)' }
+# $tag_size_max - Maximum size of an individual field.
Field will be truncated
+# if it is larger than this. Default: 2048
+# $logline_line_scratch_size - Size of static log line buffer. If a
line is larger than
+# this buffer, temp buffers will be
allocated. Set this
+# slighly larger than your expected line
size.
+# Default: 4096
+# $logline_hash_size - Number of hash buckets. Set this to
avg_requests_per_second / 5.
+# Default: 5000
+# $logline_hash_max - Max number of log lines / bucket. Set
this to
+# avg_requests_per_second / $log_hash_size.
+# Default: 5
+# $logline_data_copy - If true, log tag data read from VSL files
+# should be copied instantly when read.
Default true.
+# $log_level - varnishkafka log level. Default 6
(info).
+# $log_stderr - Boolean. Whether to log to stderr.
Default: true
+# $log_syslog - Boolean. Whether to log to syslog.
Default: true
+# $log_statistics_file - Path to varnishkafka JSON statistics
file.
+# Default:
/var/cache/varnishkafka/varnishkafka.stats.json
+# $log_statistics_interval - JSON statistics file output interval in
seconds. Default: 60
+#
+# $should_subscribe - If true, the varnishkafka service will
restart for config
+# changes. Default: true.
+# $conf_template
+#
+define varnishkafka::instance(
+ $brokers = ['localhost:9092'],
+ $topic = 'varnish',
+ $sequence_number = 0,
+ $output = 'kafka',
+ $format_type = 'string',
+ $format = '%l %n %t
%{Varnish:time_firstbyte}x %h %{Varnish:handling}x/%s %b %m
http://%{Host}i%U%q - %{Content-Type}o %{Referer}i
%{X-Forwarded-For}i %{User-agent!escape}i %{Accept-Language}i',
+ $format_key_type = 'string',
+ $format_key = undef,
+
+ $partition = -1,
+ $queue_buffering_max_messages = 100000,
+ $queue_buffering_max_ms = 1000,
+ $batch_num_messages = 1000,
+ $message_send_max_retries = 3,
+ $topic_request_required_acks = 1,
+ $topic_message_timeout_ms = 300000,
+ $topic_request_timeout_ms = 5000,
+ $socket_send_buffer_bytes = 0,
+ $compression_codec = 'none',
+
+ $varnish_name = undef,
+ $varnish_opts = { 'm' => 'RxRequest:^(?!PURGE$)', },
+
+ $tag_size_max = 2048,
+ $logline_scratch_size = 4096,
+ $logline_hash_size = 5000,
+ $logline_hash_max = 5,
+ $logline_data_copy = true,
+
+ $log_level = 6,
+ $log_stderr = false,
+ $log_syslog = true,
+ $log_statistics_file =
"/var/cache/varnishkafka/${name}.stats.json",
+ $log_statistics_interval = 60,
+
+ $should_subscribe = true,
+ $conf_template = 'varnishkafka/varnishkafka.conf.erb',
+) {
+ require ::varnishkafka
+
+ file { "/etc/varnishkafka/${name}.conf":
+ content => template($conf_template),
+ require => Package['varnishkafka'],
+ }
+
+ file { "/etc/init/varnishkafka-${name}.conf":
+ content => template('varnishkafka/varnishkafka.upstart.conf.erb'),
+ require => Package['varnishkafka'],
+ }
+
+ service { "varnishkafka-${name}":
+ ensure => 'running',
+ enable => true,
+ hasstatus => true,
+ hasrestart => true,
+ }
+
+ # subscribe varnishkafka to its config files
+ if $should_subscribe {
+ File["/etc/varnishkafka/${name}.conf"] ~>
Service["varnishkafka-${name}"]
+ }
+ # else just require them
+ else {
+ File["/etc/varnishkafka/${name}.conf"] ->
Service["varnishkafka-${name}"]
+ }
+}
diff --git a/manifests/monitor.pp b/manifests/monitor.pp
new file mode 100644
index 0000000..3ce1a03
--- /dev/null
+++ b/manifests/monitor.pp
@@ -0,0 +1,38 @@
+# == Define varnishkafka::monitor
+# Installs varnishkafka python ganglia module.
+#
+define varnishkafka::monitor(
+ $log_statistics_file = "/var/cache/varnishkafka/${name}.stats.json",
+ $log_statistics_interval = 60,
+) {
+ require ::varnishkafka
+
+ Varnishkafka::Instance[$name] -> Varnishkafka::Monitor[$name]
+
+ if ! defined(File['/usr/lib/ganglia/python_modules/varnishkafka.py']) {
+ file { '/usr/lib/ganglia/python_modules/varnishkafka.py':
+ source =>
'puppet:///modules/varnishkafka/varnishkafka_ganglia.py',
+ require => Package['ganglia-monitor'],
+ notify => Service['gmond'],
+ }
+ }
+
+ # Metrics reported by varnishkafka_ganglia.py are
+ # not known until the varnishkafka.stats.json file is
+ # parsed. Run it with the --generate-pyconf option to
+ # generate the .pyconf file now.
+ exec { "generate-varnishkafka-${name}.pyconf":
+ require => File['/usr/lib/ganglia/python_modules/varnishkafka.py'],
+ command => "/usr/bin/python
/usr/lib/ganglia/python_modules/varnishkafka.py --generate
--tmax=${log_statistics_interval} ${log_statistics_file} >
/etc/ganglia/conf.d/varnishkafka-${name}.pyconf.new",
+ onlyif => "/usr/bin/test -s ${log_statistics_file}",
+ }
+
+ exec { "replace-varnishkafka-${name}.pyconf":
+ cwd => '/etc/ganglia/conf.d',
+ path => '/bin:/usr/bin',
+ unless => "diff -q varnishkafka-${name}.pyconf.new
varnishkafka-${name}.pyconf && rm varnishkafka-${name}.pyconf.new",
+ command => "mv varnishkafka-${name}.pyconf.new
varnishkafka-${name}.pyconf",
+ require => Exec["generate-varnishkafka-${name}.pyconf"],
+ notify => Service['gmond'],
+ }
+}
diff --git a/manifests/monitoring.pp b/manifests/monitoring.pp
deleted file mode 100644
index 1cd6866..0000000
--- a/manifests/monitoring.pp
+++ /dev/null
@@ -1,36 +0,0 @@
-# == Class varnishkafka::monitoring
-# Installs varnishkafka python ganglia module.
-#
-class varnishkafka::monitoring(
- $ensure = 'present'
-)
-{
- Class['varnishkafka'] -> Class['varnishkafka::monitoring']
-
- $log_statistics_file = $::varnishkafka::log_statistics_file
- $log_statistics_interval = $::varnishkafka::log_statistics_interval
- file { '/usr/lib/ganglia/python_modules/varnishkafka.py':
- source => 'puppet:///modules/varnishkafka/varnishkafka_ganglia.py',
- require => Package['ganglia-monitor'],
- notify => Service['gmond'],
- }
-
- # Metrics reported by varnishkafka_ganglia.py are
- # not known until the varnishkafka.stats.json file is
- # parsed. Run it with the --generate-pyconf option to
- # generate the .pyconf file now.
- exec { 'generate-varnishkafka.pyconf':
- require => File['/usr/lib/ganglia/python_modules/varnishkafka.py'],
- command => "/usr/bin/python
/usr/lib/ganglia/python_modules/varnishkafka.py --generate
--tmax=${log_statistics_interval} ${log_statistics_file} >
/etc/ganglia/conf.d/varnishkafka.pyconf.new",
- onlyif => "/usr/bin/test -f ${log_statistics_file}",
- }
-
- exec { 'replace-varnishkafka.pyconf':
- cwd => '/etc/ganglia/conf.d',
- path => '/bin:/usr/bin',
- unless => 'diff -q varnishkafka.pyconf.new varnishkafka.pyconf && rm
varnishkafka.pyconf.new',
- command => 'mv varnishkafka.pyconf.new varnishkafka.pyconf',
- require => Exec['generate-varnishkafka.pyconf'],
- notify => Service['gmond'],
- }
-}
diff --git a/templates/varnishkafka.default.erb
b/templates/varnishkafka.default.erb
deleted file mode 100644
index 3ab1a29..0000000
--- a/templates/varnishkafka.default.erb
+++ /dev/null
@@ -1,7 +0,0 @@
-# Note: This file is managed by Puppet.
-
-# Configuration file for varnishkafka
-#
-# This shell script fragment is sourced by /etc/init.d/varnishkafka
-#
-VARNISHKAFKA_ENABLED=1
diff --git a/templates/varnishkafka.upstart.conf.erb
b/templates/varnishkafka.upstart.conf.erb
new file mode 100644
index 0000000..184d6b2
--- /dev/null
+++ b/templates/varnishkafka.upstart.conf.erb
@@ -0,0 +1,15 @@
+# varnishkafka - Varnish log collector with Apache Kafka integration
+
+description "Varnish log collector"
+
+start on (local-filesystems and net-device-up IFACE!=lo)
+
+setuid varnishlog
+setgid varnishlog
+
+exec /usr/bin/varnishkafka -S "/etc/varnishkafka/<%= @name %>.conf"
+
+respawn
+respawn limit unlimited
+
+# vim: set ft=upstart:
--
To view, visit https://gerrit.wikimedia.org/r/172418
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I205f92c72e3da23771ce2223b6eb8ee48ac95c5b
Gerrit-PatchSet: 11
Gerrit-Project: operations/puppet/varnishkafka
Gerrit-Branch: master
Gerrit-Owner: Ori.livneh <[email protected]>
Gerrit-Reviewer: Giuseppe Lavagetto <[email protected]>
Gerrit-Reviewer: Ori.livneh <[email protected]>
Gerrit-Reviewer: Ottomata <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
