Ori.livneh has uploaded a new change for review.
https://gerrit.wikimedia.org/r/172418
Change subject: Allow multiple instances
......................................................................
Allow multiple instances
Allow multiple varnishkafka instances per node by changing the ::varnishkafka
class to a custom resource type -- ::varnishkafa::instance.
Each instance is managed via an upstart config
/etc/init/varnishkafka-${name}.conf
and a config file /etc/varnishkafka/${name}.conf.
Untested!
Change-Id: I205f92c72e3da23771ce2223b6eb8ee48ac95c5b
---
R files/varnishkafka.default
M manifests/defaults.pp
M manifests/init.pp
A manifests/instance.pp
A templates/varnishkafka.upstart.conf.erb
5 files changed, 179 insertions(+), 135 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/operations/puppet/varnishkafka
refs/changes/18/172418/1
diff --git a/templates/varnishkafka.default.erb b/files/varnishkafka.default
similarity index 100%
rename from templates/varnishkafka.default.erb
rename to files/varnishkafka.default
diff --git a/manifests/defaults.pp b/manifests/defaults.pp
index a8f76a9..9a3d2fb 100644
--- a/manifests/defaults.pp
+++ b/manifests/defaults.pp
@@ -42,5 +42,4 @@
$should_subscribe = true
$conf_template = 'varnishkafka/varnishkafka.conf.erb'
- $default_template = 'varnishkafka/varnishkafka.default.erb'
}
diff --git a/manifests/init.pp b/manifests/init.pp
index 21f3593..a32ff9e 100644
--- a/manifests/init.pp
+++ b/manifests/init.pp
@@ -1,149 +1,34 @@
# == Class varnishkafka
# Configures and runs varnishkafka Varnish to Kafka producer.
# See: https://github.com/wikimedia/varnishkafka
-# Most varnishkafka.conf properties are supported.
#
-# == Parameters
-# $brokers - Array of Kafka broker host:ports.
-# Default: [localhost:9091]
-# $topic - Kafka topic name to produce to.
-# Default: varnish
-# $sequence_number - Sequence number at which to start
logging.
-# You can set this to an arbitrary
integer, or to
-# 'time', which will start the sequence
number
-# at the current timestamp * 10000000.
Default: 0
-# $output - output type. Either 'kafka', 'stdout',
or 'null'.
-# Default: kafka
-# $format_type - Log format type. Either 'string' or
'json'
-# Default: string
-# $format - Log format string.
-# $format_key_type - Kafka message key format type.
-# Either 'string' or 'json'. Default:
string
-# $format_key - Kafka message key format string.
-# Default: undef (disables Kafka message
key usage).
-# $partition - Topic partition number to send to. -1
for random.
-# Default: -1
-# $queue_buffering_max_messages - Maximum number of messages allowed on the
-# local Kafka producer queue. Default:
100000
-# $queue_buffering_max_ms - Maximum time, in milliseconds, for
buffering
-# data on the producer queue. Default:
1000
-# $batch_num_messages - Maximum number of messages batched in
one MessageSet.
-# Default: 1000
-# $message_send_max_retries - Maximum number of retries per messageset.
-# Default: 3
-# $topic_request_required_acks - Required ack level. Default: 1
-# $topic_message_timeout_ms - Local message timeout (milliseconds).
-# Default: 300000
-# $topic_request_timeout_ms - Ack timeout of the produce request.
-# Default: 5000
-# $socket_send_buffer_bytes - SO_SNDBUFF Socket send buffer size.
System default is used if 0.
-# Default: 0
-# $compression_codec - Compression codec to use when sending
batched messages to
-# Kafka. Valid values are 'none', 'gzip',
and 'snappy'.
-# Default: none
-# $varnish_name - Name of varnish instance to log from.
Default: undef
-# $varnish_opts - Arbitrary hash of varnish CLI options.
-# Default: { 'm' =>
'RxRequest:^(?!PURGE$)' }
-# $tag_size_max - Maximum size of an individual field.
Field will be truncated
-# if it is larger than this. Default: 2048
-# $logline_line_scratch_size - Size of static log line buffer. If a
line is larger than
-# this buffer, temp buffers will be
allocated. Set this
-# slighly larger than your expected line
size.
-# Default: 4096
-# $logline_hash_size - Number of hash buckets. Set this to
avg_requests_per_second / 5.
-# Default: 5000
-# $logline_hash_max - Max number of log lines / bucket. Set
this to
-# avg_requests_per_second / $log_hash_size.
-# Default: 5
-# $logline_data_copy - If true, log tag data read from VSL files
-# should be copied instantly when read.
Default true.
-# $log_level - varnishkafka log level. Default 6
(info).
-# $log_stderr - Boolean. Whether to log to stderr.
Default: true
-# $log_syslog - Boolean. Whether to log to syslog.
Default: true
-# $log_statistics_file - Path to varnishkafka JSON statistics
file.
-# Default:
/var/cache/varnishkafka/varnishkafka.stats.json
-# $log_statistics_interval - JSON statistics file output interval in
seconds. Default: 60
-#
-# $should_subscribe - If true, the varnishkafka service will
restart for config
-# changes. Default: true.
-# $conf_template
-# $default_template
-#
-class varnishkafka(
- $brokers = $varnishkafka::defaults::brokers,
- $topic = $varnishkafka::defaults::topic,
- $sequence_number = $varnishkafka::defaults::sequence_number,
-
- $output = $varnishkafka::defaults::output,
- $format_type = $varnishkafka::defaults::format_type,
- $format = $varnishkafka::defaults::format,
-
- $format_key_type = $varnishkafka::defaults::format_key_type,
- $format_key = $varnishkafka::defaults::format_key,
-
- $partition = $varnishkafka::defaults::partition,
- $queue_buffering_max_messages =
$varnishkafka::defaults::queue_buffering_max_messages,
- $queue_buffering_max_ms =
$varnishkafka::defaults::queue_buffering_max_ms,
- $batch_num_messages =
$varnishkafka::defaults::batch_num_messages,
-
- $message_send_max_retries =
$varnishkafka::defaults::message_send_max_retries,
- $topic_request_required_acks =
$varnishkafka::defaults::topic_request_required_acks,
- $topic_message_timeout_ms =
$varnishkafka::defaults::topic_message_timeout_ms,
- $topic_request_timeout_ms =
$varnishkafka::defaults::topic_request_timeout_ms,
- $socket_send_buffer_bytes =
$varnishkafka::defaults::socket_send_buffer_bytes,
- $compression_codec =
$varnishkafka::defaults::compression_codec,
-
- $varnish_opts = $varnishkafka::defaults::varnish_opts,
- $varnish_name = $varnishkafka::defaults::varnish_name,
-
- $tag_size_max = $varnishkafka::defaults::tag_size_max,
- $logline_scratch_size =
$varnishkafka::defaults::logline_scratch_size,
- $logline_hash_size =
$varnishkafka::defaults::logline_hash_size,
- $logline_hash_max =
$varnishkafka::defaults::logline_hash_max,
- $logline_data_copy =
$varnishkafka::defaults::logline_data_copy,
-
- $log_level = $varnishkafka::defaults::log_level,
- $log_stderr = $varnishkafka::defaults::log_stderr,
- $log_syslog = $varnishkafka::defaults::log_syslog,
- $log_statistics_file =
$varnishkafka::defaults::log_statistics_file,
- $log_statistics_interval =
$varnishkafka::defaults::log_statistics_interval,
-
- $should_subscribe =
$varnishkafka::defaults::should_subscribe,
- $conf_template = $varnishkafka::defaults::conf_template,
- $default_template = $varnishkafka::defaults::default_template
-) inherits varnishkafka::defaults
-{
+class varnishkafka {
package { 'varnishkafka':
- ensure => 'present',
+ ensure => present,
}
- file { '/etc/varnishkafka.conf':
- content => template($conf_template),
- require => Package['varnishkafka'],
+ file { '/etc/varnishkafka':
+ ensure => directory,
+ owner => 'root',
+ group => 'root',
+ mode => '0444',
+ recurse => true,
+ purge => true,
+ force => true,
}
+
+ # Don't use the init script provided by the package, because it precludes
+ # running multiple instances.
file { '/etc/default/varnishkafka':
- content => template($default_template),
+ source => 'puppet:///modules/varnishkafka/varnishkafka.default',
require => Package['varnishkafka'],
}
service { 'varnishkafka':
- ensure => 'running',
- enable => true,
- hasstatus => true,
- hasrestart => true,
+ ensure => stopped,
+ provider => 'debian',
+ enable => false,
+ require => Package['varnishkafka'],
}
-
- # subscribe varnishkafka to its config files
- if $should_subscribe {
- Service['varnishkafka'] {
- subscribe => [File['/etc/varnishkafka.conf'],
File['/etc/default/varnishkafka']],
- }
- }
- # else just require them
- else {
- Service['varnishkafka'] {
- require => [File['/etc/varnishkafka.conf'],
File['/etc/default/varnishkafka']],
- }
- }
-}
\ No newline at end of file
+}
diff --git a/manifests/instance.pp b/manifests/instance.pp
new file mode 100644
index 0000000..2098fcc
--- /dev/null
+++ b/manifests/instance.pp
@@ -0,0 +1,145 @@
+# == Class varnishkafka
+# Configures and runs varnishkafka Varnish to Kafka producer.
+# See: https://github.com/wikimedia/varnishkafka
+# Most varnishkafka.conf properties are supported.
+#
+# == Parameters
+# $brokers - Array of Kafka broker host:ports.
+# Default: [localhost:9091]
+# $topic - Kafka topic name to produce to.
+# Default: varnish
+# $sequence_number - Sequence number at which to start
logging.
+# You can set this to an arbitrary
integer, or to
+# 'time', which will start the sequence
number
+# at the current timestamp * 10000000.
Default: 0
+# $output - output type. Either 'kafka', 'stdout',
or 'null'.
+# Default: kafka
+# $format_type - Log format type. Either 'string' or
'json'
+# Default: string
+# $format - Log format string.
+# $format_key_type - Kafka message key format type.
+# Either 'string' or 'json'. Default:
string
+# $format_key - Kafka message key format string.
+# Default: undef (disables Kafka message
key usage).
+# $partition - Topic partition number to send to. -1
for random.
+# Default: -1
+# $queue_buffering_max_messages - Maximum number of messages allowed on the
+# local Kafka producer queue. Default:
100000
+# $queue_buffering_max_ms - Maximum time, in milliseconds, for
buffering
+# data on the producer queue. Default:
1000
+# $batch_num_messages - Maximum number of messages batched in
one MessageSet.
+# Default: 1000
+# $message_send_max_retries - Maximum number of retries per messageset.
+# Default: 3
+# $topic_request_required_acks - Required ack level. Default: 1
+# $topic_message_timeout_ms - Local message timeout (milliseconds).
+# Default: 300000
+# $topic_request_timeout_ms - Ack timeout of the produce request.
+# Default: 5000
+# $socket_send_buffer_bytes - SO_SNDBUFF Socket send buffer size.
System default is used if 0.
+# Default: 0
+# $compression_codec - Compression codec to use when sending
batched messages to
+# Kafka. Valid values are 'none', 'gzip',
and 'snappy'.
+# Default: none
+# $varnish_name - Name of varnish instance to log from.
Default: undef
+# $varnish_opts - Arbitrary hash of varnish CLI options.
+# Default: { 'm' =>
'RxRequest:^(?!PURGE$)' }
+# $tag_size_max - Maximum size of an individual field.
Field will be truncated
+# if it is larger than this. Default: 2048
+# $logline_line_scratch_size - Size of static log line buffer. If a
line is larger than
+# this buffer, temp buffers will be
allocated. Set this
+# slighly larger than your expected line
size.
+# Default: 4096
+# $logline_hash_size - Number of hash buckets. Set this to
avg_requests_per_second / 5.
+# Default: 5000
+# $logline_hash_max - Max number of log lines / bucket. Set
this to
+# avg_requests_per_second / $log_hash_size.
+# Default: 5
+# $logline_data_copy - If true, log tag data read from VSL files
+# should be copied instantly when read.
Default true.
+# $log_level - varnishkafka log level. Default 6
(info).
+# $log_stderr - Boolean. Whether to log to stderr.
Default: true
+# $log_syslog - Boolean. Whether to log to syslog.
Default: true
+# $log_statistics_file - Path to varnishkafka JSON statistics
file.
+# Default:
/var/cache/varnishkafka/varnishkafka.stats.json
+# $log_statistics_interval - JSON statistics file output interval in
seconds. Default: 60
+#
+# $should_subscribe - If true, the varnishkafka service will
restart for config
+# changes. Default: true.
+# $conf_template
+#
+define varnishkafka::instance(
+ $brokers = $varnishkafka::defaults::brokers,
+ $topic = $varnishkafka::defaults::topic,
+ $sequence_number = $varnishkafka::defaults::sequence_number,
+
+ $output = $varnishkafka::defaults::output,
+ $format_type = $varnishkafka::defaults::format_type,
+ $format = $varnishkafka::defaults::format,
+
+ $format_key_type = $varnishkafka::defaults::format_key_type,
+ $format_key = $varnishkafka::defaults::format_key,
+
+ $partition = $varnishkafka::defaults::partition,
+ $queue_buffering_max_messages =
$varnishkafka::defaults::queue_buffering_max_messages,
+ $queue_buffering_max_ms =
$varnishkafka::defaults::queue_buffering_max_ms,
+ $batch_num_messages =
$varnishkafka::defaults::batch_num_messages,
+
+ $message_send_max_retries =
$varnishkafka::defaults::message_send_max_retries,
+ $topic_request_required_acks =
$varnishkafka::defaults::topic_request_required_acks,
+ $topic_message_timeout_ms =
$varnishkafka::defaults::topic_message_timeout_ms,
+ $topic_request_timeout_ms =
$varnishkafka::defaults::topic_request_timeout_ms,
+ $socket_send_buffer_bytes =
$varnishkafka::defaults::socket_send_buffer_bytes,
+ $compression_codec =
$varnishkafka::defaults::compression_codec,
+
+ $varnish_opts = $varnishkafka::defaults::varnish_opts,
+ $varnish_name = $varnishkafka::defaults::varnish_name,
+
+ $tag_size_max = $varnishkafka::defaults::tag_size_max,
+ $logline_scratch_size =
$varnishkafka::defaults::logline_scratch_size,
+ $logline_hash_size =
$varnishkafka::defaults::logline_hash_size,
+ $logline_hash_max =
$varnishkafka::defaults::logline_hash_max,
+ $logline_data_copy =
$varnishkafka::defaults::logline_data_copy,
+
+ $log_level = $varnishkafka::defaults::log_level,
+ $log_stderr = $varnishkafka::defaults::log_stderr,
+ $log_syslog = $varnishkafka::defaults::log_syslog,
+ $log_statistics_file =
$varnishkafka::defaults::log_statistics_file,
+ $log_statistics_interval =
$varnishkafka::defaults::log_statistics_interval,
+
+ $should_subscribe =
$varnishkafka::defaults::should_subscribe,
+ $conf_template = $varnishkafka::defaults::conf_template,
+) {
+ require ::varnishkafka
+ require ::varnishkafka::defaults
+
+ file { "/etc/varnishkafka/${name}.conf":
+ content => template($conf_template),
+ require => Package['varnishkafka'],
+ }
+
+ file { "/etc/init/varnishkafka-${name}.conf":
+ content => template('varnishkafka/varnishkafka.upstart.conf.erb'),
+ require => Package['varnishkafka'],
+ }
+
+ service { "varnishkafka-${name}":
+ ensure => 'running',
+ enable => true,
+ hasstatus => true,
+ hasrestart => true,
+ }
+
+ # subscribe varnishkafka to its config files
+ if $should_subscribe {
+ Service['varnishkafka'] {
+ subscribe => File["/etc/varnishkafka/${name}.conf"],
+ }
+ }
+ # else just require them
+ else {
+ Service['varnishkafka'] {
+ require => File["/etc/varnishkafka/${name}.conf"],
+ }
+ }
+}
diff --git a/templates/varnishkafka.upstart.conf.erb
b/templates/varnishkafka.upstart.conf.erb
new file mode 100644
index 0000000..46d69cb
--- /dev/null
+++ b/templates/varnishkafka.upstart.conf.erb
@@ -0,0 +1,15 @@
+# varnishkafka - Varnish log collector with Apache Kafka integration
+
+description "Varnish log collector"
+
+start on (local-filesystems and net-device-up IFACE!=lo)
+
+setuid varnishlog
+setgid varnishlog
+
+exec /usr/bin/varnishkafka -S "/etc/varnishkafka/<%= @instance_name %>.conf"
+
+respawn
+respawn limit unlimited
+
+# vim: set ft=upstart:
--
To view, visit https://gerrit.wikimedia.org/r/172418
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I205f92c72e3da23771ce2223b6eb8ee48ac95c5b
Gerrit-PatchSet: 1
Gerrit-Project: operations/puppet/varnishkafka
Gerrit-Branch: master
Gerrit-Owner: Ori.livneh <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
