Filippo Giunchedi has submitted this change and it was merged. Change subject: logstash: Count MediaWiki log events with statsd ......................................................................
logstash: Count MediaWiki log events with statsd Count each MediaWiki logstash event by sending an increment command for the counter "logstash.rate.mediawiki.$CHANNEL.$LEVEL" (eg "logstash.rate.mediawiki.memcached.ERROR") to a statsd server. Bug: T100735 Change-Id: I4771abef68a151d08340b63f95cc556c8010416d --- M hieradata/labs/deployment-prep/common.yaml M hieradata/role/common/logstash.yaml M manifests/role/logstash.pp A modules/logstash/manifests/output/statsd.pp A modules/logstash/templates/output/statsd.erb 5 files changed, 107 insertions(+), 1 deletion(-) Approvals: Filippo Giunchedi: Verified; Looks good to me, approved jenkins-bot: Verified diff --git a/hieradata/labs/deployment-prep/common.yaml b/hieradata/labs/deployment-prep/common.yaml index 8f98702..e7a7293 100644 --- a/hieradata/labs/deployment-prep/common.yaml +++ b/hieradata/labs/deployment-prep/common.yaml @@ -133,6 +133,7 @@ "elasticsearch::cluster_name": beta-search "elasticsearch::script_disable_dynamic": false "elasticsearch::auto_create_index": '+apifeatureusage-*,-*' +role::logstash::statsd_host: labmon1001.eqiad.wmnet nfs_mounts: project: false home: false diff --git a/hieradata/role/common/logstash.yaml b/hieradata/role/common/logstash.yaml index f32924a..0a1363c 100644 --- a/hieradata/role/common/logstash.yaml +++ b/hieradata/role/common/logstash.yaml @@ -38,6 +38,8 @@ - logstash1005.eqiad.wmnet - logstash1006.eqiad.wmnet +role::logstash::statsd_host: statsd.eqiad.wmnet + # Kibana role::kibana::vhost: logstash.wmflabs.org role::kibana::serveradmin: [email protected] diff --git a/manifests/role/logstash.pp b/manifests/role/logstash.pp index 2834399..3892ce9 100644 --- a/manifests/role/logstash.pp +++ b/manifests/role/logstash.pp @@ -5,7 +5,12 @@ # # Provisions Logstash and ElasticSearch. # -class role::logstash { +# == Parameters: +# - $statsd_host: Host to send statsd data to. +# +class role::logstash ( + $statsd_host, +) { include ::role::logstash::elasticsearch include ::logstash @@ -71,6 +76,7 @@ priority => 70, } + ## Outputs (90) # Template for Elasticsearch index creation file { '/etc/logstash/elasticsearch-template.json': ensure => present, @@ -88,6 +94,14 @@ template => '/etc/logstash/elasticsearch-template.json', require => File['/etc/logstash/elasticsearch-template.json'], } + + logstash::output::statsd { 'MW_channel_rate': + host => $statsd_host, + guard_condition => '[type] == "mediawiki" and "es" in [tags]', + namespace => 'logstash.rate', + sender => 'mediawiki', + increment => [ '%{channel}.%{level}' ], + } } # == Class: role::logstash::elasticsearch diff --git a/modules/logstash/manifests/output/statsd.pp b/modules/logstash/manifests/output/statsd.pp new file mode 100644 index 0000000..82dd2ea --- /dev/null +++ b/modules/logstash/manifests/output/statsd.pp @@ -0,0 +1,60 @@ +# == Define: logstash::output::statsd +# +# Configure logstash to output to statsd +# +# Metric names are formed as "${namespace}.${sender}.${metric}" following the +# Etsy naming style. $namespace will be omitted if set to an empty string and +# can be formed using Logstash "%{foo}" style printf substution based on the +# contents of the Logstash event being processed. $sender will have dots +# replaced with underscores. These conventions are enforced by the Logstash +# output plugin, so figure out how deal with them for your use case. +# +# == Parameters: +# - $ensure: Whether the config should exist. Default present. +# - $host: statsd server. Default '127.0.0.1'. +# - $port: statsd server port. Default 8125. +# - $guard_condition: Logstash condition to require to pass events to output. +# Default undef. +# - $namespace: The statsd namespace to use for this metric. +# Default 'logstash'. +# - $sender: Name of the sender. Dots will be replaced with underscores. +# Default $host. +# - $count: Hash of metric_name => count values. Default undef. +# - $decrement: Array of metric names to decrement. Default undef. +# - $gauge: Hash of metric_name => gauge values. Default undef. +# - $increment: Array of metric names to increment. Default undef. +# - $set: Hash of metric_name => set values. Default undef. +# - $timing: Hash of metric_name => timing values. Default undef. +# - $sample_rate: The sample rate for the metric. Default 1. +# +# == Sample usage: +# +# logstash::output::statsd { 'MW_channel_rate': +# guard_condition => '[type] == "mediawiki" and "es" in [tags]', +# namespace => 'logstash.rate', +# sender => 'mediawiki', +# increment => [ "%{channel}.%{level}" ], +# } +# +define logstash::output::statsd( + $ensure = present, + $host = '127.0.0.1', + $port = 8125, + $guard_condition = undef, + $namespace = 'logstash', + $sender = $host, + $count = undef, + $decrement = undef, + $gauge = undef, + $increment = undef, + $set = undef, + $timing = undef, + $sample_rate = 1, +) { + logstash::conf { "output-statsd-${title}": + ensure => $ensure, + content => template('logstash/output/statsd.erb'), + priority => $priority, + } +} +# vim:sw=4 ts=4 sts=4 et: diff --git a/modules/logstash/templates/output/statsd.erb b/modules/logstash/templates/output/statsd.erb new file mode 100644 index 0000000..a6ec11f --- /dev/null +++ b/modules/logstash/templates/output/statsd.erb @@ -0,0 +1,29 @@ +output { +<% if @guard_condition %>if <%= @guard_condition %> {<% end %> + statsd { + host => "<%= @host %>" + port => <%= @port %> + namespace => "<%= @namespace %>" + sender => "<%= @sender %>" + sample_rate => <%= @sample_rate %> + <% if @count %> + count => [ "<%= @count.map{|e| e.join('","') }.join('","') %>" ] + <% end %> + <% if @decrement %> + decrement => [ "<%= @decrement.join('","') %>" ] + <% end %> + <% if @gauge %> + gauge => [ "<%= @gauge.map{|e| e.join('","') }.join('","') %>" ] + <% end %> + <% if @increment %> + increment => [ "<%= @increment.join('","') %>" ] + <% end %> + <% if @set %> + set => [ "<%= @set.map{|e| e.join('","') }.join('","') %>" ] + <% end %> + <% if @timing %> + timing => [ "<%= @timing.map{|e| e.join('","') }.join('","') %>" ] + <% end %> + } +<% if @guard_condition %>}<% end %> +} -- To view, visit https://gerrit.wikimedia.org/r/230233 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I4771abef68a151d08340b63f95cc556c8010416d Gerrit-PatchSet: 6 Gerrit-Project: operations/puppet Gerrit-Branch: production Gerrit-Owner: BryanDavis <[email protected]> Gerrit-Reviewer: BryanDavis <[email protected]> Gerrit-Reviewer: Filippo Giunchedi <[email protected]> Gerrit-Reviewer: Ori.livneh <[email protected]> Gerrit-Reviewer: jenkins-bot <> _______________________________________________ MediaWiki-commits mailing list [email protected] https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
