Ori.livneh has uploaded a new change for review. https://gerrit.wikimedia.org/r/73388
Change subject: Rewrite of EventLogging module ...................................................................... Rewrite of EventLogging module This commit is the first in a series of commits that aim to comprehensively refactor the EventLogging Puppet module. The refactoring migrates process management of EventLogging daemons from Supervisord to Upstart and abstracts site-specific configurations out of the module itself. The best Puppet modules use Puppet abstractions to provide a mini-DSL for managing a service class, and that is what I have attempted to do here. The module provides a suite of four custom resource types. Each type corresponds to a particular kind of daemonized actor that performs some data processing step by transforming the event stream in some definite way. These resources can be freely composed into a data processing pipeline. The provisioned services are glued together using a publisher/subscribe messaging pattern over TCP/IP, which can scale to an arbitrary number of instances on an arbitrary number of hosts. This change provides the bulk of the implementation and most of the clean-up. Change-Id: I4f32cb4cbd54b6121855ea6430051b41191c4c4a --- A manifests/role/eventlogging.pp M manifests/role/logging.pp A modules/eventlogging/README A modules/eventlogging/files/eventloggingctl A modules/eventlogging/files/init/consumer.conf A modules/eventlogging/files/init/forwarder.conf A modules/eventlogging/files/init/init.conf A modules/eventlogging/files/init/multiplexer.conf A modules/eventlogging/files/init/processor.conf D modules/eventlogging/files/supervisord.conf A modules/eventlogging/manifests/decommission.pp A modules/eventlogging/manifests/deployment.pp M modules/eventlogging/manifests/init.pp R modules/eventlogging/manifests/monitor.pp A modules/eventlogging/manifests/packages.pp A modules/eventlogging/manifests/service/consumer.pp A modules/eventlogging/manifests/service/forwarder.pp A modules/eventlogging/manifests/service/multiplexer.pp A modules/eventlogging/manifests/service/processor.pp D modules/eventlogging/manifests/supervisor.pp A modules/eventlogging/templates/consumer.erb A modules/eventlogging/templates/forwarder.erb A modules/eventlogging/templates/multiplexer.erb A modules/eventlogging/templates/processor.erb 24 files changed, 535 insertions(+), 104 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/operations/puppet refs/changes/88/73388/1 diff --git a/manifests/role/eventlogging.pp b/manifests/role/eventlogging.pp new file mode 100644 index 0000000..d193b3a --- /dev/null +++ b/manifests/role/eventlogging.pp @@ -0,0 +1,76 @@ +# == Class: role::eventlogging +# This role configures an instance to act as the primary EventLogging +# log processor for the cluster. The setup is described in detail on +# <https://wikitech.wikimedia.org/wiki/EventLogging>. +# +# XXX(ori-l, 3-Jul-2013): Document this better. +# +class role::eventlogging { + system_role { 'misc::log-collector': + description => 'EventLogging log collector', + } + + include eventlogging + include eventlogging::mediawiki_errors + include eventlogging::monitor + + class { 'eventlogging::archive': + destinations => [ 'stat1.wikimedia.org', 'stat1002.eqiad.wmnet' ], + } + + class { 'mongodb': + dbpath => '/srv/mongodb', + bind_ip => false, + auth => true, + } + + require passwords::mongodb::eventlogging # RT 5101 + $mongo_user = $passwords::mongodb::eventlogging::user + $mongo_pass = $passwords::mongodb::eventlogging::password + + require passwords::mysql::eventlogging # RT 4752 + $mysql_user = $passwords::mysql::eventlogging::user + $mysql_pass = $passwords::mysql::eventlogging::password + + eventlogging::service::forwarder { + '8421': + ensure => present, + count => true; + '8422': + ensure => present; + } + + eventlogging::service::processor { + 'server-side events': + format => '%n EventLogging %j', + input => 'tcp://localhost:8421', + output => 'tcp://*:8521'; + 'client-side events': + format => '%q %l %n %t %h', + input => 'tcp://localhost:8422', + output => 'tcp://*:8522'; + } + + eventlogging::service::multiplexer { 'all events': + inputs => [ 'tcp://127.0.0.1:8521', 'tcp://127.0.0.1:8522' ], + output => 'tcp://*:8600', + } + + eventlogging::service::consumer { + 'vanadium': + input => 'tcp://vanadium.eqiad.wmnet:8600', + output => "mongodb://${mongo_user}:${mongo_pass}@vanadium.eqiad.wmnet:27017/?w=1"; + 'mysql-db1047': + input => 'tcp://vanadium.eqiad.wmnet:8600', + output => "mysql://${mysql_user}:${mysql_pass}@db1047.eqiad.wmnet/log?charset=utf8"; + 'server-side-events.log': + input => 'tcp://vanadium.eqiad.wmnet:8421', + output => 'file:///var/log/eventlogging/server-side-events.log'; + 'client-side-events.log': + input => 'tcp://vanadium.eqiad.wmnet:8422', + output => 'file:///var/log/eventlogging/client-side-events.log'; + 'all-events.log': + input => 'tcp://vanadium.eqiad.wmnet:8600', + output => 'file:///var/log/eventlogging/all-events.log'; + } +} diff --git a/manifests/role/logging.pp b/manifests/role/logging.pp index a652c67..df133f5 100644 --- a/manifests/role/logging.pp +++ b/manifests/role/logging.pp @@ -290,16 +290,3 @@ monitor_packet_loss => false, } } - - -# EventLogging collector -class role::logging::eventlogging { - system_role { "misc::log-collector": - description => 'EventLogging log collector', - } - deployment::target { "eventlogging": } - - class { '::eventlogging': - archive_destinations => [ 'stat1.wikimedia.org', 'stat1002.eqiad.wmnet' ], - } -} diff --git a/modules/eventlogging/README b/modules/eventlogging/README new file mode 100644 index 0000000..805c09b --- /dev/null +++ b/modules/eventlogging/README @@ -0,0 +1,18 @@ +EventLogging +------------ +EventLogging is a platform for modeling, logging and processing +arbitrary analytic data. This Puppet module manages the configuration +of its event processing component, which comprises a suite of four +services, each of which implements a different step in a data +processing pipeline. The services communicate with one another using a +publisher/subscriber model, using ØMQ as the transport layer. +An arbitrary number of service instances running on an arbitrary +number of hosts can be freely composed to form different event +processing patterns. + +For more information, see the extension's page and user guide on +MediaWiki.org and the maintenance instructions on Wikitech: + + * https://www.mediawiki.org/wiki/Extension:EventLogging + * https://www.mediawiki.org/wiki/Extension:EventLogging/Guide + * https://wikitech.wikimedia.org/wiki/EventLogging diff --git a/modules/eventlogging/files/eventloggingctl b/modules/eventlogging/files/eventloggingctl new file mode 100755 index 0000000..e0c50ed --- /dev/null +++ b/modules/eventlogging/files/eventloggingctl @@ -0,0 +1,24 @@ +#!/bin/bash +# eventloggingctl -- Manage EventLogging services +# Usage: eventloggingctl {start|stop|restart|status|tail} +# +command=$1 +shift +case "$command" in + status) + initctl list | grep -Po '(?<=eventlogging/)(?!init).*' | sort -k5 \ + | sed 's/, process//' | column -ts'( )' + ;; + start|restart) + initctl emit eventlogging.start + ;; + stop) + initctl emit eventlogging.stop + ;; + tail) + tail "$@" /var/log/upstart/eventlogging_init.log + ;; + *) + echo >&2 "Usage: ${0##*/} {start|restart|stop|status|tail}" + ;; +esac diff --git a/modules/eventlogging/files/init/consumer.conf b/modules/eventlogging/files/init/consumer.conf new file mode 100644 index 0000000..0773276 --- /dev/null +++ b/modules/eventlogging/files/init/consumer.conf @@ -0,0 +1,19 @@ +# eventlogging/consumer +# +# This is an Upstart job configuration file, describing an EventLogging +# consumer service job. EventLogging consumers subscribe to an event +# stream and log it to some medium. +# +description "EventLogging consumer" +author "Ori Livneh <[email protected]>" + +instance $NAME + +stop on eventlogging.stop + +setuid eventlogging +setgid eventlogging + +exec eventlogging-consumer "@$CONFIG" + +respawn diff --git a/modules/eventlogging/files/init/forwarder.conf b/modules/eventlogging/files/init/forwarder.conf new file mode 100644 index 0000000..01f92a0 --- /dev/null +++ b/modules/eventlogging/files/init/forwarder.conf @@ -0,0 +1,18 @@ +# eventlogging/forwarder +# +# This service listens on an UDP port and forwards all incoming data to a +# ZeroMQ PUB socket bound to the TCP port of the same number. +# +description "EventLogging UDP-to-ZMQ forwarder" +author "Ori Livneh <[email protected]>" + +instance $NAME + +stop on eventlogging.stop + +setuid eventlogging +setgid eventlogging + +exec eventlogging-forwarder "@$CONFIG" + +respawn diff --git a/modules/eventlogging/files/init/init.conf b/modules/eventlogging/files/init/init.conf new file mode 100644 index 0000000..1d800cf --- /dev/null +++ b/modules/eventlogging/files/init/init.conf @@ -0,0 +1,27 @@ +# eventlogging/init +# +# This is an Upstart job configuration file, describing a task job +# (<upstart.ubuntu.com/cookbook/#task-job>). Its role is to start +# service jobs representing the different components of an EventLogging +# server An EventLogging server receives raw log data via UDP and +# transforms it into a sanitized & validated ZeroMQ event stream. +# +description "Initialize EventLogging" +author "Ori Livneh <[email protected]>" + +start on runlevel [2345] or eventlogging.start + +task + +script + roles="forwarder processor multiplexer consumer" + set -- $roles + for role in "$@"; do + for config in /etc/eventlogging.d/${role}s/*; do + [ -e "$config" ] || break + name="$(basename $config)" + start eventlogging/$role NAME="$name" CONFIG="$config" || + status eventlogging/$role NAME="$name" + done + done +end script diff --git a/modules/eventlogging/files/init/multiplexer.conf b/modules/eventlogging/files/init/multiplexer.conf new file mode 100644 index 0000000..19265fc --- /dev/null +++ b/modules/eventlogging/files/init/multiplexer.conf @@ -0,0 +1,19 @@ +# eventlogging/multiplexer +# +# This is an Upstart job configuration file, describing a service that +# subscribes to multiple ZeroMQ publishers and multiplexes them into a +# single output stream. +# +description "EventLogging ZeroMQ Multiplexer" +author "Ori Livneh <[email protected]>" + +instance $NAME + +stop on eventlogging.stop + +setuid eventlogging +setgid eventlogging + +exec eventlogging-multiplexer "@$CONFIG" + +respawn diff --git a/modules/eventlogging/files/init/processor.conf b/modules/eventlogging/files/init/processor.conf new file mode 100644 index 0000000..a82c6ab --- /dev/null +++ b/modules/eventlogging/files/init/processor.conf @@ -0,0 +1,19 @@ +# eventlogging/processor +# +# This is an Upstart job configuration file, describing an EventLogging stream +# processor. An EventLogging stream processor parses and validates raw log +# data, transforming it into a JSON event stream. +# +description "EventLogging stream processor" +author "Ori Livneh <[email protected]>" + +instance $NAME + +stop on eventlogging.stop + +setuid eventlogging +setgid eventlogging + +exec eventlogging-processor "@$CONFIG" + +respawn diff --git a/modules/eventlogging/files/supervisord.conf b/modules/eventlogging/files/supervisord.conf deleted file mode 100644 index 9e435da..0000000 --- a/modules/eventlogging/files/supervisord.conf +++ /dev/null @@ -1,26 +0,0 @@ -; supervisor config -; ----------------- -; Managed by Puppet: puppet:///files/eventlogging/eventlogging.conf -; For detailed explanations of these configuration variables, see the -; supervisor docs at <http://supervisord.org/configuration.html>. - -[unix_http_server] -file = /var/run//supervisor.sock - -[supervisord] -logfile = /var/log/supervisor/supervisord.log -pidfile = /var/run/supervisord.pid -childlogdir = /var/log/supervisor -environment = PYTHONIOENCODING=utf-8:ignore,PYTHONPATH=/srv/deployment/eventlogging/EventLogging/server,PATH=/srv/deployment/eventlogging/EventLogging/server/bin - -[rpcinterface:supervisor] -supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface - -[supervisorctl] -serverurl = unix:///var/run//supervisor.sock - -[inet_http_server] -port = *:2828 - -[include] -files = /etc/supervisor/conf.d/*.conf diff --git a/modules/eventlogging/manifests/decommission.pp b/modules/eventlogging/manifests/decommission.pp new file mode 100644 index 0000000..74ffa1d --- /dev/null +++ b/modules/eventlogging/manifests/decommission.pp @@ -0,0 +1,21 @@ +# == Class: eventlogging::decommission +# +# This class automates the process of retiring an EventLogging node. It +# shuts down EventLogging services and removes their configuration data. +# +class eventlogging::decommission { + exec { 'initctl emit eventlogging.stop': + before => File['/etc/init/eventlogging', '/etc/eventlogging.d'], + } + + file { [ '/etc/init/eventlogging', '/etc/eventlogging.d' ]: + ensure => absent, + purge => true, + force => true, + recurse => true, + } + + deployment::target { 'eventlogging': + ensure => absent, + } +} diff --git a/modules/eventlogging/manifests/deployment.pp b/modules/eventlogging/manifests/deployment.pp new file mode 100644 index 0000000..f3a8f5d --- /dev/null +++ b/modules/eventlogging/manifests/deployment.pp @@ -0,0 +1,11 @@ +# == Class: eventlogging::deployment +# +# This class manages the git-deploy configuration for EventLogging. +# +class eventlogging::deployment { + $path = '/srv/deployment/eventlogging/EventLogging' + + deployment::target { 'eventlogging': + before => File['/etc/init/eventlogging'], + } +} diff --git a/modules/eventlogging/manifests/init.pp b/modules/eventlogging/manifests/init.pp index 6031163..b9ec695 100644 --- a/modules/eventlogging/manifests/init.pp +++ b/modules/eventlogging/manifests/init.pp @@ -1,35 +1,58 @@ -# Collector of analytic events -# See <https://wikitech.wikimedia.org/wiki/EventLogging> -class eventlogging( $archive_destinations = [] ) { +# == Class: eventlogging +# +# EventLogging is a platform for modeling, logging and processing +# arbitrary analytic data. This Puppet module manages the configuration +# of its event processing component, which comprises a suite of four +# services, each of which implements a different step in a data +# processing pipeline. The services communicate with one another using a +# publisher/subscriber model, using ØMQ as the transport layer. +# An arbitrary number of service instances running on an arbitrary +# number of hosts can be freely composed to form different event +# processing patterns. +# +class eventlogging { + include eventlogging::package - class { 'eventlogging::supervisor': } - class { 'eventlogging::analysis': } - class { 'eventlogging::ganglia': } - class { 'eventlogging::archive': - destinations => $archive_destinations, - } - - class { 'mongodb': - dbpath => '/srv/mongodb', - bind_ip => false, - auth => true, - } - - class { 'eventlogging::mediawiki_errors': } - - package { [ - 'python-jsonschema', - 'python-mysqldb', - 'python-pygments', - 'python-pymongo', - 'python-sqlalchemy', - 'python-zmq', - ]: + group { 'eventlogging': ensure => present, } - systemuser { 'eventlogging': - name => 'eventlogging', + user { 'eventlogging': + ensure => present, + gid => 'eventlogging', + shell => '/bin/false', + home => '/srv/eventlogging', + managehome => true, + system => true, } + file { [ + '/etc/eventlogging.d', + '/etc/eventlogging.d/consumers', + '/etc/eventlogging.d/forwarders', + '/etc/eventlogging.d/multiplexers', + '/etc/eventlogging.d/processors' + ]: + ensure => directory, + before => File['/etc/init/eventlogging'], + } + + + file { '/etc/init/eventlogging': + source => 'puppet:///modules/eventlogging/init', + recurse => true, + } + + file { '/sbin/eventloggingctl': + source => 'puppet:///modules/eventlogging/eventloggingctl', + mode => '0755', + } + + service { 'eventlogging/init': + provider => 'upstart', + require => [ + File['/etc/init/eventlogging'], + User['eventlogging'] + ], + } } diff --git a/modules/eventlogging/manifests/ganglia.pp b/modules/eventlogging/manifests/monitor.pp similarity index 75% rename from modules/eventlogging/manifests/ganglia.pp rename to modules/eventlogging/manifests/monitor.pp index 876fca1..bbcb4a3 100644 --- a/modules/eventlogging/manifests/ganglia.pp +++ b/modules/eventlogging/manifests/monitor.pp @@ -1,9 +1,10 @@ # Monitor events per second using Ganglia -class eventlogging::ganglia { +class eventlogging::monitor { + include eventlogging file { '/usr/lib/ganglia/python_modules/zpubmon.py': ensure => link, - target => '/srv/deployment/eventlogging/EventLogging/ganglia/python_modules/zpubmon.py', + target => "${eventlogging::path}/ganglia/python_modules/zpubmon.py", require => [ File['/usr/lib/ganglia/python_modules'], Package['python-zmq'], @@ -17,7 +18,6 @@ File['/etc/ganglia/conf.d'], File['/usr/lib/ganglia/python_modules/zpubmon.py'], ], - notify => Service[gmond], + notify => Service['gmond'], } - } diff --git a/modules/eventlogging/manifests/packages.pp b/modules/eventlogging/manifests/packages.pp new file mode 100644 index 0000000..94924a2 --- /dev/null +++ b/modules/eventlogging/manifests/packages.pp @@ -0,0 +1,24 @@ +# == Class: eventlogging::package +# +# This class configures a host for having EventLogging software deployed on it. +# As of July 2013, EventLogging's dependencies are available in apt, but +# EventLogging itself is distributed via git-deploy. +# +class eventlogging::package { + $path = '/srv/deployment/eventlogging/EventLogging' + + package { [ + 'python-jsonschema', + 'python-mysqldb', + 'python-pygments', + 'python-pymongo', + 'python-sqlalchemy', + 'python-zmq', + ]: + ensure => present, + } + + deployment::target { 'eventlogging': + before => File['/etc/init/eventlogging'], + } +} diff --git a/modules/eventlogging/manifests/service/consumer.pp b/modules/eventlogging/manifests/service/consumer.pp new file mode 100644 index 0000000..43b7f50 --- /dev/null +++ b/modules/eventlogging/manifests/service/consumer.pp @@ -0,0 +1,50 @@ +# == Define: eventlogging::service::consumer +# +# Consumers are data sinks; they act as an interface between +# EventLogging and various data storage, monitoring, and visualization +# systems. Multiple consumers may subscribe to a single event stream. +# One consumer may write the data to Hadoop, another to statsd, another +# to MongoDB, etc. Both the input stream and the output medium are +# specified by URI. A plug-in architecture provides a mechanism for a +# plug-in to register itself as the handler of a particular output URI +# scheme (for example, the MongoDB consumer handles "mongo://" URIs). +# +# === Parameters +# +# [*input*] +# This parameter specifies the URI of the event stream the consumer +# should consume. Example: 'tcp://vanadium.eqiad.wmnet:8600'. +# +# [*output*] +# Bind the multiplexing publisher to this URI. +# Example: 'tcp://*:8600'. +# +# [*sid*] +# Specifies the Socket ID consumer should use to identify itself when +# subscribing to the input stream. Defaults to the resource title. +# +# [*ensure*] +# Specifies whether the consumer should be provisioned or destroyed. +# Value may be 'present' (provisions the resource; the default) or +# 'absent' (destroys the resource). +# +# === Examples +# +# eventlogging::service::consumer { 'all events': +# input => 'tcp://vanadium.eqiad.wmnet:8600', +# output => 'mongodb://vanadium.eqiad.wmnet:27017/?w=1', +# } +# +define eventlogging::service::consumer( + $input, + $output, + $sid = $title, + $ensure = present, +) { + $basename = regsubst($title, '\W', '-', 'G') + file { "/etc/eventlogging.d/consumers/${basename}": + ensure => $ensure, + content => template('eventlogging/consumer.erb'), + notify => Service['eventlogging/init'], + } +} diff --git a/modules/eventlogging/manifests/service/forwarder.pp b/modules/eventlogging/manifests/service/forwarder.pp new file mode 100644 index 0000000..b9a388d --- /dev/null +++ b/modules/eventlogging/manifests/service/forwarder.pp @@ -0,0 +1,35 @@ +# == Define: eventlogging::service::forwarder +# +# An EventLogging forwarder listens for data on an inbound UDP port and +# publishes that data on a ZeroMQ PUB socket that is bound to the same +# port number, TCP. +# +# === Parameters +# +# [*port*] +# Port which should be forwarded. Defaults to the resource title. +# +# [*count*] +# If true, prepend an autoincrementing ID to each message that is +# forwarded. False by default. +# +# [*ensure*] +# If 'present' (the default), enable the service; if 'absent', disable +# and remove it. +# +# === Examples +# +# eventlogging::service::forwarder { '8600': ] +# +define eventlogging::service::forwarder( + $port = $title, + $count = false, + $ensure = present, +) { + $basename = regsubst($title, '\W', '-', 'G') + file { "/etc/eventlogging.d/forwarders/${basename}": + ensure => $ensure, + content => template('eventlogging/forwarder.erb'), + notify => Service['eventlogging/init'], + } +} diff --git a/modules/eventlogging/manifests/service/multiplexer.pp b/modules/eventlogging/manifests/service/multiplexer.pp new file mode 100644 index 0000000..61afee3 --- /dev/null +++ b/modules/eventlogging/manifests/service/multiplexer.pp @@ -0,0 +1,43 @@ +# == Define: eventlogging::service::multiplexer +# +# An EventLogging multiplexer forwards multiple ZeroMQ inputs into a +# single ZeroMQ publisher. +# +# === Parameters +# +# [*inputs*] +# An array of URIs for ZeroMQ publishers that should be selected as +# input. Example: ['tcp://127.0.0.1:8521', 'tcp://127.0.0.1:8522']. +# +# [*output*] +# Bind the multiplexing publisher to this URI. +# Example: 'tcp://*:8600'. +# +# [*sid*] +# Socket ID multiplexer should use to identify itself when subscribing +# to input streams. Defaults to the resource title. +# +# [*ensure*] +# If 'present' (the default), sets up the multiplexer. If 'absent', +# destroys it. +# +# === Examples +# +# eventlogging::service::multiplexer { 'all_events': +# inputs => [ 'tcp://127.0.0.1:8521', 'tcp://127.0.0.1:8522' ], +# output => 'tcp://*:8600', +# } +# +define eventlogging::service::multiplexer( + $inputs, + $output, + $sid = $title, + $ensure = present, +) { + $basename = regsubst($title, '\W', '-', 'G') + file { "/etc/eventlogging.d/multiplexers/${basename}": + ensure => $ensure, + content => template('eventlogging/multiplexer.erb'), + notify => Service['eventlogging/init'], + } +} diff --git a/modules/eventlogging/manifests/service/processor.pp b/modules/eventlogging/manifests/service/processor.pp new file mode 100644 index 0000000..2275dbb --- /dev/null +++ b/modules/eventlogging/manifests/service/processor.pp @@ -0,0 +1,61 @@ +# == Define: eventlogging::service::processor +# +# EventLogging processors transform a raw request log, such as might be +# generated by Varnish or MediaWiki, into a well-structured stream of +# JSON events. +# +# === Parameters +# +# [*format*] +# scanf-like format string, specifying the layout of EventLogging +# fields in raw log records. The available format specifiers are: +# +# %h Client IP +# %j JSON object +# %l Hostname of origin +# %n Sequence ID +# %q Query-string-encoded JSON +# %t Timestamp in NCSA format +# +# All other parts of the format string are interpreted as Python +# regexp syntax. See <http://docs.python.org/2/library/re.html> +# for details. +# +# [*input*] +# The URI of the raw log stream which the processor will take as its +# input. Example: 'tcp://vanadium.eqiad.wmnet:8421'. +# +# [*output*] +# A URI specifying the interface and port on which the processed event +# stream will be published. Example: 'tcp://*:8600'. +# +# [*sid*] +# Specifies the socket ID the processor will use to identify itself +# when subscribing to input streams. Defaults to the resource title. +# +# [*ensure*] +# If 'present' (the default), sets up the multiplexer. If 'absent', +# destroys it. +# +# === Examples +# +# eventlogging::service::processor { 'client_side_events': +# input => 'tcp://127.0.0.1:8422', +# format => '%q %l %n %t %h', +# output => 'tcp://*:8522', +# } +# +define eventlogging::service::processor( + $format, + $input, + $output, + $sid = $title, + $ensure = present, +) { + $basename = regsubst($title, '\W', '-', 'G') + file { "/etc/eventlogging.d/processors/${basename}": + ensure => $ensure, + content => template('eventlogging/processor.erb'), + notify => Service['eventlogging/init'], + } +} diff --git a/modules/eventlogging/manifests/supervisor.pp b/modules/eventlogging/manifests/supervisor.pp deleted file mode 100644 index 109ed22..0000000 --- a/modules/eventlogging/manifests/supervisor.pp +++ /dev/null @@ -1,33 +0,0 @@ -# Use supervisor process management tool (a cousin of upstart and -# daemontools) to start / stop / restart / monitor EventLogging processes. -class eventlogging::supervisor { - - require passwords::mysql::eventlogging # RT 4752 - require passwords::mongodb::eventlogging # RT 5101 - - package { 'supervisor': - ensure => present, - } - - service { 'supervisor': - ensure => running, - enable => true, - require => Package['supervisor'], - } - - file { '/etc/supervisor/supervisord.conf': - source => 'puppet:///modules/eventlogging/supervisord.conf', - require => [ Package['supervisor'], Systemuser['eventlogging'] ], - notify => Service['supervisor'], - mode => '0444', - } - - # Temporarily disabled while I upgrade EL back-end (ori-l, 17-Jun-2013): - # file { '/etc/supervisor/conf.d/eventlogging.conf': - # content => template('eventlogging/eventlogging.conf.erb'), - # require => File['/etc/supervisor/supervisord.conf'], - # notify => Service['supervisor'], - # mode => '0444', - # } - -} diff --git a/modules/eventlogging/templates/consumer.erb b/modules/eventlogging/templates/consumer.erb new file mode 100644 index 0000000..43e1747 --- /dev/null +++ b/modules/eventlogging/templates/consumer.erb @@ -0,0 +1,4 @@ +--sid +<%= @sid %> +<%= @input %> +<%= @output %> diff --git a/modules/eventlogging/templates/forwarder.erb b/modules/eventlogging/templates/forwarder.erb new file mode 100644 index 0000000..71c2297 --- /dev/null +++ b/modules/eventlogging/templates/forwarder.erb @@ -0,0 +1,2 @@ +<%= port %><% if @count %> +--count<% end %> diff --git a/modules/eventlogging/templates/multiplexer.erb b/modules/eventlogging/templates/multiplexer.erb new file mode 100644 index 0000000..4fcc024 --- /dev/null +++ b/modules/eventlogging/templates/multiplexer.erb @@ -0,0 +1,4 @@ +--sid +<%= @sid %> +<%= @output %> +<%= @inputs.join("\n") %> diff --git a/modules/eventlogging/templates/processor.erb b/modules/eventlogging/templates/processor.erb new file mode 100644 index 0000000..4e99cd6 --- /dev/null +++ b/modules/eventlogging/templates/processor.erb @@ -0,0 +1,5 @@ +--sid +<%= @sid %> +<%= @format %> +<%= @input %> +<%= @output %> -- To view, visit https://gerrit.wikimedia.org/r/73388 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I4f32cb4cbd54b6121855ea6430051b41191c4c4a Gerrit-PatchSet: 1 Gerrit-Project: operations/puppet Gerrit-Branch: production Gerrit-Owner: Ori.livneh <[email protected]> _______________________________________________ MediaWiki-commits mailing list [email protected] https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
