Ori.livneh has uploaded a new change for review. https://gerrit.wikimedia.org/r/73997
Change subject: Document 'eventlogging' role ...................................................................... Document 'eventlogging' role This patch provides additional documentation and re-orders resource declarations to provide a description of EventLogging data flow. It does not include any configuration changes. Change-Id: I3cc4edefcbadb3a3cf336e5a30a2dd3ca302d7ee --- M manifests/role/eventlogging.pp 1 file changed, 101 insertions(+), 45 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/operations/puppet refs/changes/97/73997/1 diff --git a/manifests/role/eventlogging.pp b/manifests/role/eventlogging.pp index 32d21f3..01c9286 100644 --- a/manifests/role/eventlogging.pp +++ b/manifests/role/eventlogging.pp @@ -1,67 +1,116 @@ # == Class: role::eventlogging +# # This role configures an instance to act as the primary EventLogging # log processor for the cluster. The setup is described in detail on -# <https://wikitech.wikimedia.org/wiki/EventLogging>. +# <https://wikitech.wikimedia.org/wiki/EventLogging>. End-user +# documentation is available in the form of a guide, located at +# <https://www.mediawiki.org/wiki/Extension:EventLogging/Guide>. # -# XXX(ori-l, 3-Jul-2013): Document this better. +# There exist two APIs for generating events: efLogServerSideEvent() in +# PHP and mw.eventLog.logEvent() in JavaScript. Events generated in PHP +# are sent by the app servers directly to vanadium on UDP port 8421. +# JavaScript-generated events are URL-encoded and sent to our servers by +# means of an HTTP/S request to bits, which a varnishncsa instance +# forwards to vanadium on port 8422. These event streams are parsed, +# validated, and multiplexed into an output stream that is published via +# ZeroMQ on TCP port 8600. Data sinks are implemented as subscribers +# that connect to this endpoint and read data into some storage medium. # class role::eventlogging { - system_role { 'misc::log-collector': - description => 'EventLogging log collector', + include eventlogging + include eventlogging::monitor + + system_role { 'role::eventlogging': + description => 'EventLogging', } - include eventlogging - include eventlogging::monitor - class { 'eventlogging::archive': - destinations => [ 'stat1.wikimedia.org', 'stat1002.eqiad.wmnet' ], + ## Data flow + + # Server-side events are generated by MediaWiki and sent to vanadium + # on UDP port 8421, using wfErrorLog('...', 'udp://...'). vanadium + # is specified as the destination in $wgEventLoggingFile, declared + # in wmf-config/CommonSettings.php. + + eventlogging::service::forwarder { '8421': + ensure => present, + count => true, + } + + eventlogging::service::processor { 'server-side events': + format => '%n EventLogging %j', + input => 'tcp://localhost:8421', + output => 'tcp://*:8521', + } + + # Client-side events are generated by JavaScript-triggered HTTP/S + # requests to //bits.wikimedia.org/event.gif?<event payload>. + # A varnishncsa instance on the bits caches greps for these requests + # and forwards them to vanadium on UDP port 8422. The varnishncsa + # configuration is specified in <manifests/role/cache.pp>. + + eventlogging::service::forwarder { '8422': + ensure => present, } - class { 'mongodb': - dbpath => '/srv/mongodb', - bind_ip => false, - auth => true, + eventlogging::service::processor { 'client-side events': + format => '%q %l %n %t %h', + input => 'tcp://localhost:8422', + output => 'tcp://*:8522', } - require passwords::mongodb::eventlogging # RT 5101 - $mongo_user = $passwords::mongodb::eventlogging::user - $mongo_pass = $passwords::mongodb::eventlogging::password - - require passwords::mysql::eventlogging # RT 4752 - $mysql_user = $passwords::mysql::eventlogging::user - $mysql_pass = $passwords::mysql::eventlogging::password - - eventlogging::service::forwarder { - '8421': - ensure => present, - count => true; - '8422': - ensure => present; - } - - eventlogging::service::processor { - 'server-side events': - format => '%n EventLogging %j', - input => 'tcp://localhost:8421', - output => 'tcp://*:8521'; - 'client-side events': - format => '%q %l %n %t %h', - input => 'tcp://localhost:8422', - output => 'tcp://*:8522'; - } + # Parsed and validated client-side (Varnish-generated) and + # server-side (MediaWiki-generated) events are multiplexed into a + # single output stream, published on TCP port 8600. eventlogging::service::multiplexer { 'all events': inputs => [ 'tcp://127.0.0.1:8521', 'tcp://127.0.0.1:8522' ], output => 'tcp://*:8600', } - eventlogging::service::consumer { - 'vanadium': - input => 'tcp://vanadium.eqiad.wmnet:8600', - output => "mongodb://${mongo_user}:${mongo_pass}@vanadium.eqiad.wmnet:27017"; - 'mysql-db1047': - input => 'tcp://vanadium.eqiad.wmnet:8600', - output => "mysql://${mysql_user}:${mysql_pass}@db1047.eqiad.wmnet/log?charset=utf8"; + + ## MongoDB + + # Log events to a local MongoDB instance. + + include passwords::mongodb::eventlogging # RT 5101 + $mongo_user = $passwords::mongodb::eventlogging::user + $mongo_pass = $passwords::mongodb::eventlogging::password + + class { 'mongodb': + dbpath => '/srv/mongodb', + bind_ip => false, + auth => true, + } + + eventlogging::service::consumer { 'vanadium': + input => 'tcp://vanadium.eqiad.wmnet:8600', + output => "mongodb://${mongo_user}:${mongo_pass}@vanadium.eqiad.wmnet:27017", + } + + + ## MySQL / MariaDB + + # Log strictly valid events to the 'log' database on db1047. This is the + # primary medium for data analysis as of July 2013. + + include passwords::mysql::eventlogging # RT 4752 + $mysql_user = $passwords::mysql::eventlogging::user + $mysql_pass = $passwords::mysql::eventlogging::password + + eventlogging::service::consumer { 'mysql-db1047': + input => 'tcp://vanadium.eqiad.wmnet:8600', + output => "mysql://${mysql_user}:${mysql_pass}@db1047.eqiad.wmnet/log?charset=utf8", + } + + + ## Flat files + + # Log all raw log records and decoded events to flat files in + # /var/log/eventlogging as a medium of last resort. These files + # are rotated and rsynced to stat1 & stat1002 for backup. + + eventlogging::service::consumer { 'server-side-events.log': input => 'tcp://vanadium.eqiad.wmnet:8421', output => 'file:///var/log/eventlogging/server-side-events.log'; @@ -72,4 +121,11 @@ input => 'tcp://vanadium.eqiad.wmnet:8600', output => 'file:///var/log/eventlogging/all-events.log'; } + + class { 'eventlogging::archive': + destinations => [ + 'stat1.wikimedia.org', + 'stat1002.eqiad.wmnet' + ], + } } -- To view, visit https://gerrit.wikimedia.org/r/73997 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I3cc4edefcbadb3a3cf336e5a30a2dd3ca302d7ee Gerrit-PatchSet: 1 Gerrit-Project: operations/puppet Gerrit-Branch: production Gerrit-Owner: Ori.livneh <[email protected]> _______________________________________________ MediaWiki-commits mailing list [email protected] https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
